def define_identifier(self): """ Return the syntax definition for an identifier. """ # --- Defining the individual identifiers: # Getting all the Unicode numbers in a single string: unicode_numbers = "".join([unichr(n) for n in xrange(0x10000) if unichr(n).isdigit()]) unicode_number_expr = Regex("[%s]" % unicode_numbers, re.UNICODE) space_char = re.escape(self._grammar.get_token("identifier_spacing")) identifier0 = Regex("[\w%s]+" % space_char, re.UNICODE) # Identifiers cannot start with a number: identifier0 = Combine(~unicode_number_expr + identifier0) identifier0.setName("individual_identifier") # --- Defining the namespaces: namespace_sep = Suppress(self._grammar.get_token("namespace_separator")) namespace = Group(ZeroOrMore(identifier0 + namespace_sep)) namespace.setName("namespace") # --- The full identifier, which could have a namespace: identifier = Combine(namespace.setResultsName("namespace_parts") + identifier0.setResultsName("identifier")) identifier.setName("full_identifier") return identifier
def define_identifier(self): """ Return the syntax definition for an identifier. """ # --- Defining the individual identifiers: # Getting all the Unicode numbers in a single string: unicode_numbers = "".join( [unichr(n) for n in xrange(0x10000) if unichr(n).isdigit()]) unicode_number_expr = Regex("[%s]" % unicode_numbers, re.UNICODE) space_char = re.escape(self._grammar.get_token("identifier_spacing")) identifier0 = Regex("[\w%s]+" % space_char, re.UNICODE) # Identifiers cannot start with a number: identifier0 = Combine(~unicode_number_expr + identifier0) identifier0.setName("individual_identifier") # --- Defining the namespaces: namespace_sep = Suppress( self._grammar.get_token("namespace_separator")) namespace = Group(ZeroOrMore(identifier0 + namespace_sep)) namespace.setName("namespace") # --- The full identifier, which could have a namespace: identifier = Combine( namespace.setResultsName("namespace_parts") + identifier0.setResultsName("identifier")) identifier.setName("full_identifier") return identifier
def define_operand(self): """ Return the syntax definition for an operand. An operand can be a variable, a string, a number or a set. A set is made of other operands, including other sets. **This method shouldn't be overridden**. Instead, override the syntax definitions for variables, strings and/or numbers. If you want to customize the sets, check :meth:`T_SET_START`, :meth:`T_SET_END` and :meth:`T_ELEMENT_SEPARATOR`. """ identifier = self.define_identifier() operand = Forward() # Defining the sets: set_start = Suppress(self._grammar.get_token("set_start")) set_end = Suppress(self._grammar.get_token("set_end")) element_separator = self._grammar.get_token("element_separator") elements = delimitedList(operand, delim=element_separator) set_ = Group(set_start + Optional(elements) + set_end) set_.setParseAction(self.make_set) set_.setName("set") # Defining the variables: variable = identifier.copy() variable.setName("variable") variable.addParseAction(self.make_variable) # Defining the functions: function_name = identifier.setResultsName("function_name") function_name.setName("function_name") args_start = Suppress(self._grammar.get_token("arguments_start")) args_end = Suppress(self._grammar.get_token("arguments_end")) args_sep = self._grammar.get_token("arguments_separator") arguments = Optional(Group(delimitedList(operand, delim=args_sep)), default=()) arguments = arguments.setResultsName("arguments") arguments.setParseAction(lambda tokens: tokens[0]) function = function_name + args_start + arguments + args_end function.setName("function") function.setParseAction(self.make_function) operand << (function | variable | self.define_number() | \ self.define_string() | set_) return operand
def define_identifier(self): """ Return the syntax definition for an identifier. """ # --- Defining the individual identifiers: # Getting all the Unicode numbers in a single string: try: unicode_numbers = "".join( [unichr(n) for n in xrange(0x10000) if unichr(n).isdigit()]) except NameError: unicode_numbers = "".join( [chr(n) for n in range(0x10000) if chr(n).isdigit()]) unicode_number_expr = Regex("[%s]" % unicode_numbers, re.UNICODE) space_char = re.escape(self._grammar.get_token("identifier_spacing")) identifier0 = Regex("[\w%s]+" % space_char, re.UNICODE) # Identifiers cannot start with a number: identifier0 = Combine(identifier0) identifier0.setName("individual_identifier") # --- Defining the namespaces: namespace_sep = Suppress( self._grammar.get_token("namespace_separator")) namespace = Group(ZeroOrMore(identifier0 + namespace_sep)) namespace.setName("namespace") # --- The full identifier, which could have a namespace: identifier = Combine( namespace.setResultsName("namespace_parts") + identifier0.setResultsName("identifier")) identifier.setName("full_identifier") expop = Literal('^') multop = oneOf('* /') factop = Literal('!') modop = Literal('%') signop = oneOf('+ -') opers = expop | signop | multop | factop | modop identifier = identifier + NotAny(opers) return identifier
_dblQuote + ZeroOrMore(CharsNotIn('\\"\n\r') | _escapedChar | '""') + _dblQuote).streamline().setName("string enclosed in double quotes") sglQuotedString = Combine( _sglQuote + ZeroOrMore(CharsNotIn("\\'\n\r") | _escapedChar | "''") + _sglQuote).streamline().setName("string enclosed in single quotes") quotedArg = (dblQuotedString | sglQuotedString) quotedArg.setParseAction(removeQuotes) quotedArg.setName("quotedArg") plainArgChars = printables.replace('#', '').replace('"', '').replace("'", "") plainArg = Word(plainArgChars) plainArg.setName("plainArg") arguments = Group(ZeroOrMore(quotedArg | plainArg)) arguments = arguments.setResultsName('arguments') arguments.setName("arguments") # comment line. comment = Literal('#') + restOfLine comment = comment.suppress() comment.setName('comment') full_command = (comment | (command + arguments + Optional(comment))) full_command.setName('full_command') ### command_list = [] # filled in by namespaces.init_global_dict(). # command/argument handling.
PrimaryExpression.setName('PrimaryExpression') UnaryExpression = ( (Suppress('!') + PrimaryExpression).setParseAction( refer_component(components.Operators.LogicalNegation)) | (Suppress('+') + PrimaryExpression).setParseAction( refer_component(components.Operators.NumericPositive)) | (Suppress('-') + PrimaryExpression).setParseAction( refer_component(components.Operators.NumericNegative)) | PrimaryExpression) if DEBUG: UnaryExpression.setName('UnaryExpression') MultiplicativeExpression = Group(UnaryExpression + ZeroOrMore( (Literal('*') | Literal('/')) + UnaryExpression)).setParseAction( refer_component(components.Expression.ParsedMultiplicativeExpressionList)) if DEBUG: MultiplicativeExpression.setName('MultiplicativeExpression') AdditiveExpression = Group(MultiplicativeExpression + ZeroOrMore( (Literal('+') | Literal('-')) + MultiplicativeExpression)).setParseAction( refer_component(components.Expression.ParsedAdditiveExpressionList)) if DEBUG: AdditiveExpression.setName('AdditiveExpression') NumericExpression = AdditiveExpression RelationalExpression = ( (NumericExpression + Suppress('=') + NumericExpression).setParseAction( refer_component(components.Operators.EqualityOperator)) | (NumericExpression + Suppress('!=') + NumericExpression).setParseAction( refer_component(components.Operators.NotEqualOperator)) | (NumericExpression + Suppress('<') + NumericExpression).setParseAction(
_dblQuote = Literal('"') _escapables = printables _escapedChar = Word(_bslash, _escapables, exact=2) dblQuotedString = Combine( _dblQuote + ZeroOrMore( CharsNotIn('\\"\n\r') | _escapedChar | '""' ) + _dblQuote ).streamline().setName("string enclosed in double quotes") sglQuotedString = Combine( _sglQuote + ZeroOrMore( CharsNotIn("\\'\n\r") | _escapedChar | "''" ) + _sglQuote ).streamline().setName("string enclosed in single quotes") quotedArg = ( dblQuotedString | sglQuotedString ) quotedArg.setParseAction(removeQuotes) quotedArg.setName("quotedArg") plainArgChars = printables.replace('#', '').replace('"', '').replace("'", "") plainArg = Word(plainArgChars) plainArg.setName("plainArg") arguments = Group(ZeroOrMore(quotedArg | plainArg)) arguments = arguments.setResultsName('arguments') arguments.setName("arguments") # comment line. comment = Literal('#') + restOfLine comment = comment.suppress() comment.setName('comment') full_command = ( comment | (command + arguments + Optional(comment)) ) full_command.setName('full_command') ### command_list = [] # filled in by namespaces.init_global_dict().
UnaryExpression = ((Suppress('!') + PrimaryExpression).setParseAction( refer_component(components.LogicalNegation)) | (Suppress('+') + PrimaryExpression).setParseAction( refer_component(components.NumericPositive)) | (Suppress('-') + PrimaryExpression).setParseAction( refer_component(components.NumericNegative)) | PrimaryExpression) if DEBUG: UnaryExpression.setName('UnaryExpression') MultiplicativeExpression = Group(UnaryExpression + ZeroOrMore( (Literal('*') | Literal('/')) + UnaryExpression)).setParseAction( refer_component(components.ParsedMultiplicativeExpressionList)) if DEBUG: MultiplicativeExpression.setName('MultiplicativeExpression') AdditiveExpression = Group(MultiplicativeExpression + ZeroOrMore( (Literal('+') | Literal('-')) + MultiplicativeExpression)).setParseAction( refer_component(components.ParsedAdditiveExpressionList)) if DEBUG: AdditiveExpression.setName('AdditiveExpression') NumericExpression = AdditiveExpression RelationalExpression = ( (NumericExpression + Suppress('=') + NumericExpression).setParseAction( refer_component(components.EqualityOperator)) | (NumericExpression + Suppress('!=') + NumericExpression).setParseAction( refer_component(components.NotEqualOperator)) | (NumericExpression + Suppress('<') + NumericExpression).setParseAction( refer_component(components.LessThanOperator)) |
def make_parser(): ParserElement.setDefaultWhitespaceChars(' \t') EOL = OneOrMore(LineEnd()).suppress().setName("end of line") Spaces = OneOrMore(" ").suppress() # NOTE: These are not all 'printable' Unicode characters. # If needed, expand the alphas_extra variable. alphas_extra = ''.join(chr(x) for x in range(0x100, 0x350)) chars = printables + alphas8bit + alphas_extra Token = Word(chars) InlineComment = '#' - SkipTo(EOL) WholelineComment = LineStart() + '#' - restOfLine - EOL Argument = Token('arg').setName('argument') Variable = Token('var').setName('variable') KindObject = Keyword('kind')('object') KindVerb = Keyword('is')('verb') Kind = Named(Keyword('url') | Keyword('raw') | Keyword('text'))('arg') MatchObject = Named(Keyword('arg'))('object') data = Named(Keyword('data'))('object') MatchVerb = Named( Keyword('is') | Keyword('istype') | Keyword('matches') | Keyword('rewrite'))('verb').setName('verb') Pattern = Named(Group(OneOrMore(Spaces + Argument + EOL)))('arg').leaveWhitespace() ActionObject = Keyword('plumb')('object') ActionVerb = Named( Keyword('run') | Keyword('notify') | Keyword('download'))('verb') Action = Named(originalTextFor(OneOrMore(Argument)))('arg') ArgMatchClause = Group(MatchObject - MatchVerb - Variable - Pattern) DataMatchClause = Group(data - MatchVerb - Pattern) # Transform every 'data match' rule to an equivalent 'arg match' rule def data_to_arg(toks): assert (len(toks) == 1) toks[0][0] = 'arg' toks[0].insert(2, '{data}') return toks DataMatchClause.setParseAction(data_to_arg) KindClause = Group(KindObject - KindVerb - Kind) - EOL MatchClause = (DataMatchClause | ArgMatchClause) ActionClause = Group(ActionObject - ActionVerb - Action) - EOL MatchBlock = Group(ZeroOrMore(MatchClause('match-clause'))) ActionBlock = Group(OneOrMore(ActionClause('action-clause'))) # TODO: allow the excluded chars if they are escaped. RuleName = Word(chars, excludeChars='{ } [ ]')('rule-name') RuleHeading = Suppress('[') - RuleName - Suppress(']') - EOL Rule = Group(RuleHeading - KindClause('kind-clause') - MatchBlock('match-block') - ActionBlock('action-block')) RulesFile = OneOrMore(Rule) RulesFile.ignore(WholelineComment) RulesFile.ignore(InlineComment) for v in [MatchObject, ActionObject]: v.setName('object') for v in [MatchVerb, ActionVerb]: v.setName('verb') Kind.setName('kind') data.setName('object') Pattern.setName('pattern') Action.setName('action or url') KindClause.setName('kind clause') MatchClause.setName('match clause') ActionClause.setName('action clause') MatchBlock.setName('match block') ActionBlock.setName('action block') Rule.setName('rule') RuleName.setName('rule name') RulesFile.setName('rules') return RulesFile
def _create_grammar(): """Create the DBC grammar. """ word = Word(printables.replace(';', '').replace(':', '')) integer = Group(Optional('-') + Word(nums)) positive_integer = Word(nums).setName('positive integer') number = Word(nums + '.Ee-+') colon = Suppress(Literal(':')) scolon = Suppress(Literal(';')) pipe = Suppress(Literal('|')) at = Suppress(Literal('@')) sign = Literal('+') | Literal('-') lp = Suppress(Literal('(')) rp = Suppress(Literal(')')) lb = Suppress(Literal('[')) rb = Suppress(Literal(']')) comma = Suppress(Literal(',')) node = Word(alphas + nums + '_-').setWhitespaceChars(' ') frame_id = Word(nums).setName('frame id') version = Group(Keyword('VERSION') - QuotedString()) version.setName(VERSION) symbol = Word(alphas + '_') + Suppress(LineEnd()) symbols = Group(Keyword('NS_') - colon - Group(ZeroOrMore(symbol))) symbols.setName('NS_') discard = Suppress(Keyword('BS_') - colon).setName('BS_') nodes = Group(Keyword('BU_') - colon - Group(ZeroOrMore(node))) nodes.setName('BU_') signal = Group( Keyword(SIGNAL) - Group(word + Optional(word)) - colon - Group(positive_integer - pipe - positive_integer - at - positive_integer - sign) - Group(lp - number - comma - number - rp) - Group(lb - number - pipe - number - rb) - QuotedString() - Group(delimitedList(node))) signal.setName(SIGNAL) message = Group( Keyword(MESSAGE) - frame_id - word - colon - positive_integer - word - Group(ZeroOrMore(signal))) message.setName(MESSAGE) event = Suppress( Keyword(EVENT) - word - colon - positive_integer - lb - number - pipe - number - rb - QuotedString() - number - number - word - node - scolon) event.setName(EVENT) comment = Group( Keyword(COMMENT) - ((Keyword(SIGNAL) - frame_id - word - QuotedString() - scolon).setName(SIGNAL) | (Keyword(MESSAGE) - frame_id - QuotedString() - scolon).setName(MESSAGE) | (Keyword(EVENT) - word - QuotedString() - scolon).setName(EVENT) | (Keyword(NODES) - word - QuotedString() - scolon).setName(NODES) | (QuotedString() - scolon).setName('QuotedString'))) comment.setName(COMMENT) attribute_definition = Group( Keyword(ATTRIBUTE_DEFINITION) - ((QuotedString()) | (Keyword(SIGNAL) | Keyword(MESSAGE) | Keyword(EVENT) | Keyword(NODES)) + QuotedString()) - word - (scolon | (Group(ZeroOrMore(Group( (comma | Empty()) + QuotedString()))) + scolon) | (Group(ZeroOrMore(number)) + scolon))) attribute_definition.setName(ATTRIBUTE_DEFINITION) attribute_definition_default = Group( Keyword(ATTRIBUTE_DEFINITION_DEFAULT) - QuotedString() - (number | QuotedString()) - scolon) attribute_definition_default.setName(ATTRIBUTE_DEFINITION_DEFAULT) attribute = Group( Keyword(ATTRIBUTE) - QuotedString() - Group( Optional((Keyword(MESSAGE) + frame_id) | (Keyword(SIGNAL) + frame_id + word) | (Keyword(NODES) + word))) - (QuotedString() | number) - scolon) attribute.setName(ATTRIBUTE) choice = Group( Keyword(CHOICE) - Group(Optional(frame_id)) - word - Group(OneOrMore(Group(integer + QuotedString()))) - scolon) choice.setName(CHOICE) value_table = Group( Keyword(VALUE_TABLE) - word - Group(OneOrMore(Group(integer + QuotedString()))) - scolon) value_table.setName(VALUE_TABLE) signal_type = Group( Keyword(SIGNAL_TYPE) - frame_id - word - colon - positive_integer - scolon) signal_type.setName(SIGNAL_TYPE) signal_multiplexer_values = Group( Keyword(SIGNAL_MULTIPLEXER_VALUES) - frame_id - word - word - Group( delimitedList(positive_integer - Suppress('-') - Suppress(positive_integer))) - scolon) signal_multiplexer_values.setName(SIGNAL_MULTIPLEXER_VALUES) message_add_sender = Group( Keyword(MESSAGE_TX_NODE) - frame_id - colon - Group(delimitedList(node)) - scolon) message_add_sender.setName(MESSAGE_TX_NODE) attribute_definition_rel = Group( Keyword(ATTRIBUTE_DEFINITION_REL) - (QuotedString() | (Keyword(NODES_REL) + QuotedString())) - word - (scolon | (Group(ZeroOrMore(Group( (comma | Empty()) + QuotedString()))) + scolon) | (Group(ZeroOrMore(number)) + scolon))) attribute_definition_rel.setName(ATTRIBUTE_DEFINITION_REL) attribute_definition_default_rel = Group( Keyword(ATTRIBUTE_DEFINITION_DEFAULT_REL) - QuotedString() - (number | QuotedString()) - scolon) attribute_definition_default_rel.setName(ATTRIBUTE_DEFINITION_DEFAULT_REL) attribute_rel = Group( Keyword(ATTRIBUTE_REL) - QuotedString() - Keyword(NODES_REL) - word - Keyword(SIGNAL) - frame_id - word - (positive_integer | QuotedString()) - scolon) attribute_rel.setName(ATTRIBUTE_REL) signal_group = Group( Keyword(SIGNAL_GROUP) - frame_id - word - integer - colon - OneOrMore(word) - scolon) signal_group.setName(SIGNAL_GROUP) entry = (message | comment | attribute | choice | attribute_definition | attribute_definition_default | attribute_rel | attribute_definition_rel | attribute_definition_default_rel | signal_group | event | message_add_sender | value_table | signal_type | signal_multiplexer_values | discard | nodes | symbols | version) frame_id.setParseAction(lambda _s, _l, t: int(t[0])) return OneOrMore(entry) + StringEnd()
) if DEBUG: PrimaryExpression.setName("PrimaryExpression") UnaryExpression = ( (Suppress("!") + PrimaryExpression).setParseAction(refer_component(components.Operators.LogicalNegation)) | (Suppress("+") + PrimaryExpression).setParseAction(refer_component(components.Operators.NumericPositive)) | (Suppress("-") + PrimaryExpression).setParseAction(refer_component(components.Operators.NumericNegative)) | PrimaryExpression ) if DEBUG: UnaryExpression.setName("UnaryExpression") MultiplicativeExpression = Group( UnaryExpression + ZeroOrMore((Literal("*") | Literal("/")) + UnaryExpression) ).setParseAction(refer_component(components.Expression.ParsedMultiplicativeExpressionList)) if DEBUG: MultiplicativeExpression.setName("MultiplicativeExpression") AdditiveExpression = Group( MultiplicativeExpression + ZeroOrMore((Literal("+") | Literal("-")) + MultiplicativeExpression) ).setParseAction(refer_component(components.Expression.ParsedAdditiveExpressionList)) if DEBUG: AdditiveExpression.setName("AdditiveExpression") NumericExpression = AdditiveExpression RelationalExpression = ( (NumericExpression + Suppress("=") + NumericExpression).setParseAction( refer_component(components.Operators.EqualityOperator) ) | (NumericExpression + Suppress("!=") + NumericExpression).setParseAction( refer_component(components.Operators.NotEqualOperator) ) | (NumericExpression + Suppress("<") + NumericExpression).setParseAction( refer_component(components.Operators.LessThanOperator)