def defineParsers(): #Enable a fast parsing mode with caching. ParserElement.enablePackrat() #end of line terminates statements, so it is not regular whitespace ParserElement.setDefaultWhitespaceChars('\t ') func_call = Forward() #forward declaration because this is a recursive rule #The "terminal" rules symbol = Word(alphas+'_-', alphanums+'_-') .setParseAction(action_symbol) q_symbol = quotedString .setParseAction(action_q_symbol) bracket_term = Literal("(").suppress() - func_call \ + Literal(")").suppress() word = symbol | q_symbol | bracket_term #The function call #Parse: "foo | bar | baz" or "foo" pipeline = (word + ZeroOrMore("|" - word)) .setParseAction(action_pipeline) #Parse "foo|bar op1 op2 op3" func_call << (pipeline - ZeroOrMore(word)) .setParseAction(action_func_call) #High level structure of program line = LineEnd() | func_call - LineEnd() #empty line or function call program = ZeroOrMore(line) + StringEnd() #multiple lines are a program #define the comments program.ignore('%' + restOfLine) #no tab expansion program.parseWithTabs() #return additional func_call parser to make testing more easy return program, func_call
def parse_enum_defs(s, parsers=None): """Parse all of the enum defs in a string, and add them to a list of type parsers :Parameters: - `s`: the string to parse - `parsers`: the dictionary of parsers to which the new ones will be added (optional) @return: a list of parsers for parsing type declarations >>> test_string = \"""# A sample file ... a foo ... bar baz ... ... typedef enum { # a comment ... START, /* another comment */ ... END ... } RUNMARK; ... ... typedef struct { ... float x; ... int y ... } GOO ... ... typedef enum { ... OAK, ... MAPLE ... } TREES; ... ... GOO 3.4 6 ... \""" >>> parsed_defs = parse_enum_defs(test_string) >>> runmark_parser = parsed_defs['RUNMARK'] >>> runmark_parser.parseString("START")[0] 'START' >>> runmark_parser.parseString("END")[0] 'END' >>> try: ... runmark_parser.parseString("41")[0] ... except: ... print "parser threw an exception" parser threw an exception >>> >>> tree_parser = parsed_defs['TREES'] >>> tree_parser.parseString("MAPLE")[0] 'MAPLE' """ type_parser = {} if parsers is None else parsers one_enum_def_parser = make_one_enum_def_parser() not_enum = ((value_name | struct_declaration_start | possible_type_name | right_brace) + restOfLine).suppress() enum_def_parser = ZeroOrMore(Group(one_enum_def_parser) | not_enum) + stringEnd enum_def_parser.ignore(hash_comment) enum_def_parser.ignore(cStyleComment) for enum in enum_def_parser.parseString(s): type_parser[enum['enum_name']] = oneOf(enum['values'].asList()) return type_parser
def gramma(): ## Tokens point = Literal('.') prefix_op = Literal('.') choice_op = Literal('+') parallel = Literal("||") | Literal("<>") #ident = Word(alphas, alphanums+'_') ratename = Word(alphas.lower(), alphanums + "_") lpar = Literal('(').suppress() rpar = Literal(')').suppress() lsqpar = Literal('[').suppress() rsqpar = Literal(']').suppress() define = Literal('=') semicol = Literal(';').suppress() col = Literal(',').suppress() number = Word(nums) integer = number floatnumber = Combine(integer + Optional(point + Optional(number))) passiverate = Word('infty') | Word('T') internalrate = Word('tau') pound = Literal('#').suppress() percent = Literal('%').suppress() peparate = (ratename | floatnumber | internalrate | passiverate).setParseAction(_check_var) peparate_indef = floatnumber | internalrate | passiverate sync = Word('<').suppress() + ratename + ZeroOrMore(col + ratename) + Word( '>').suppress() coop_op = (parallel | sync).setParseAction(_create_sync_set) activity = (ratename + col + peparate).setParseAction(_create_activity) procdef = (Word(alphas.upper(), alphanums + "_") + Optional(lsqpar + peparate_indef + rsqpar)).setParseAction(_create_procdef) ## RATES Definitions ratedef = (Optional(percent) + ratename + define + peparate_indef).setParseAction(assign_var) + semicol prefix = Forward() choice = Forward() coop = Forward() process = (activity | procdef | lpar + coop + rpar).setParseAction(_create_process) prefix << (process + ZeroOrMore(prefix_op + prefix)).setParseAction(_create_prefix) choice << (prefix + ZeroOrMore(choice_op + choice)).setParseAction(_create_choice) coop << (choice + ZeroOrMore(coop_op + coop)).setParseAction(_create_coop) rmdef = (Optional(pound) + procdef + define + coop + semicol).setParseAction(_create_definition) system_eq = Optional(pound) + coop pepa = ZeroOrMore(ratedef) + ZeroOrMore(rmdef) + system_eq.setParseAction( _create_system_equation) pepacomment = '//' + restOfLine pepa.ignore(pepacomment) return pepa
def parse_header(s, enum_def_parser, struct_def_parser, struct_parsers): """Create a dictionary of keyword assignments in a yanny par file :Parameters: - `s`: the string to parse - `enum_def_parser`: a parser that parses one enum definition - `struct_def_parser`: a parser that parses on struct definition - `struct_parsers`: a dictionary of structures that parse structure data @return: a dictionary with the keyword assignments >>> test_string = \"""# A sample file ... a foo ... bar baz goo # no more ... ... typedef enum { ... START, ... END ... } RUNMARK; ... ... typedef struct { ... float x; ... int y ... } GOO ... ... typedef enum { ... OAK, ... MAPLE ... } TREES; ... ... GOO 3.4 6 ... \""" >>> >>> enum_def_parser = make_one_enum_def_parser() >>> type_parsers = parse_enum_defs(test_string, base_type_parsers) >>> struct_def_parser = make_one_struct_def_parser(type_parsers) >>> structs, struct_parsers = parse_struct_defs(test_string, type_parsers) >>> h = parse_header(test_string, enum_def_parser, struct_def_parser, struct_parsers) >>> print h['a'] foo >>> print h['bar'] baz goo """ one_header_assignment_parser = make_one_header_assignment_parser( struct_parsers) not_header = Or([enum_def_parser, struct_def_parser] + \ [struct_parsers[k] for k in struct_parsers.keys()]).suppress() header_parser = ZeroOrMore(not_header | one_header_assignment_parser) + stringEnd header_parser.ignore(hash_comment) header_parser.ignore(cStyleComment) header = {} for d in header_parser.parseString(s): header[d['name']] = d['value'].partition('#')[0].lstrip().rstrip() return header
def _parse_items(self, source): ParserElement.setDefaultWhitespaceChars(' \t\r') EOL = LineEnd().suppress() comment = Literal('#') + Optional( restOfLine ) + EOL string = CharsNotIn("\n") line = Group( Word(alphanums + '-')('key') + Literal(':').suppress() + Optional(Combine(string + ZeroOrMore(EOL + Literal(' ') + string)))("value") + EOL ) group = ZeroOrMore(line) group.ignore(comment) return group.parseString(source, True)
def parser(): rule = Forward() body = OneOrMore(CharsNotIn('{};') + ';') sel = CharsNotIn('{};') rule <<= sel + Group( '{' + ZeroOrMore( rule | body ) + '}' ) rule.setParseAction( make_action(Rule) ) stylesheet = ZeroOrMore( rule ) stylesheet.ignore( cStyleComment ) return stylesheet
def create_grammar(): global arrows global stereotypes assert len(arrows) > 0 assert len(stereotypes) > 0 linechars = ''.join((c for c in printables if c not in '}\n')) + ' \t' norbracket = ''.join((c for c in printables if c != ']')) + ' \t' nogt = ''.join((c for c in printables if c != '>')) + ' \t' norparen = ''.join((c for c in printables if c != ')')) + ' \t' line = Word(linechars) cls_body = Group(ZeroOrMore(line)) classkeyword = Keyword('class').setResultsName('type') st_names = stereotypes.keys() st = Literal(st_names[0]) for s in st_names[1:]: st = st | Literal(s) stereotype = Group( Optional(Literal('<<').suppress() + st + Literal('>>').suppress())) identifier_list = Word(alphas) + ZeroOrMore( Literal(',').suppress() + Word(alphas)) baseclasses = Group(Optional(Literal(':').suppress() + identifier_list)) cls = Group(stereotype + classkeyword + Word(alphas) + baseclasses + \ Literal('{').suppress() + cls_body + Literal('}').suppress()) arrow_names = arrows.keys() arrow_names.sort(lambda x, y: -cmp(len(x), len(y))) arrow = Keyword(arrow_names[0]) for ar in arrow_names[1:]: arrow = arrow | Keyword(ar) relation_caption = Literal('(').suppress() + Word(norparen) + \ Literal(')').suppress() quantifier = Literal('[').suppress() + Word(norbracket) + Literal( ']').suppress() relation = Group( Word(alphas) + Group(Optional(quantifier)) + \ arrow.setResultsName('type') + \ Word(alphas) + Group(Optional(quantifier)) + \ Group(Optional(relation_caption)) ) grammar = ZeroOrMore(cls | relation) grammar.ignore(cStyleComment) grammar.ignore("//" + restOfLine) return grammar
def build_parser(): key = Word(alphanums).setResultsName('key') value = restOfLine.setParseAction(lambda string, location, tokens: tokens[ 0].strip()).setResultsName('value') property_ = Group(key + Suppress(Literal('=')) + value) properties = Group(OneOrMore(property_)).setResultsName('properties') section_name = (Suppress('[') + OneOrMore(CharsNotIn(']')) + Suppress(']')).setResultsName('section') section = Group(section_name + properties) ini_file = ZeroOrMore(section).setResultsName('sections') ini_file.ignore(pythonStyleComment) return ini_file
def build_parser(): key = Word(alphanums).setResultsName('key') value = restOfLine.setParseAction( lambda string, location, tokens: tokens[0].strip() ).setResultsName('value') property_ = Group(key + Suppress(Literal('=')) + value) properties = Group(OneOrMore(property_)).setResultsName('properties') section_name = (Suppress('[') + OneOrMore(CharsNotIn(']')) + Suppress(']')).setResultsName('section') section = Group(section_name + properties) ini_file = ZeroOrMore(section).setResultsName('sections') ini_file.ignore(pythonStyleComment) return ini_file
def create_grammar(): global arrows global stereotypes assert len(arrows) > 0 assert len(stereotypes) > 0 linechars = ''.join((c for c in printables if c not in '}\n')) + ' \t' norbracket = ''.join((c for c in printables if c != ']')) + ' \t' nogt = ''.join((c for c in printables if c != '>')) + ' \t' norparen = ''.join((c for c in printables if c != ')')) + ' \t' line = Word(linechars) cls_body = Group(ZeroOrMore(line)) classkeyword = Keyword('class').setResultsName('type') st_names = stereotypes.keys() st = Literal(st_names[0]) for s in st_names[1:]: st = st | Literal(s) stereotype = Group(Optional(Literal('<<').suppress() + st + Literal('>>').suppress())) identifier_list = Word(alphas) + ZeroOrMore(Literal(',').suppress() + Word(alphas)) baseclasses = Group(Optional(Literal(':').suppress() + identifier_list)) cls = Group(stereotype + classkeyword + Word(alphas) + baseclasses + \ Literal('{').suppress() + cls_body + Literal('}').suppress()) arrow_names = arrows.keys() arrow_names.sort(lambda x,y: -cmp(len(x), len(y))) arrow = Keyword(arrow_names[0]) for ar in arrow_names[1:]: arrow = arrow | Keyword(ar) relation_caption = Literal('(').suppress() + Word(norparen) + \ Literal(')').suppress() quantifier = Literal('[').suppress() + Word(norbracket) + Literal(']').suppress() relation = Group( Word(alphas) + Group(Optional(quantifier)) + \ arrow.setResultsName('type') + \ Word(alphas) + Group(Optional(quantifier)) + \ Group(Optional(relation_caption)) ) grammar = ZeroOrMore(cls | relation) grammar.ignore(cStyleComment) grammar.ignore("//" + restOfLine) return grammar
def getkw_bnf(self): sect_begin = Literal("{").suppress() sect_end = Literal("}").suppress() array_begin = Literal("[").suppress() array_end = Literal("]").suppress() tag_begin = Literal("<").suppress() tag_end = Literal(">").suppress() eql = Literal("=").suppress() dmark = Literal('$').suppress() end_data=Literal('$end').suppress() prtable = alphanums+r'!$%&*+-./<>?@^_|~' ival=Regex('[-]?\d+') dval=Regex('-?\d+\.\d*([eE]?[+-]?\d+)?') lval=Regex('([Yy]es|[Nn]o|[Tt]rue|[Ff]alse|[Oo]n|[Oo]ff)') # Helper definitions kstr= quotedString.setParseAction(removeQuotes) ^ \ dval ^ ival ^ lval ^ Word(prtable) name = Word(alphas+"_",alphanums+"_") vec=array_begin+delimitedList(dval ^ ival ^ lval ^ Word(prtable) ^ \ Literal("\n").suppress() ^ \ quotedString.setParseAction(removeQuotes))+array_end sect=name+sect_begin tag_sect=name+Group(tag_begin+name+tag_end)+sect_begin # Grammar keyword = name + eql + kstr vector = name + eql + vec data=Combine(dmark+name)+SkipTo(end_data)+end_data section=Forward() sect_def=(sect | tag_sect ) #| vec_sect) input=section | data | vector | keyword section << sect_def+ZeroOrMore(input) + sect_end # Parsing actions ival.setParseAction(self.conv_ival) dval.setParseAction(self.conv_dval) lval.setParseAction(self.conv_lval) keyword.setParseAction(self.store_key) vector.setParseAction(self.store_vector) data.setParseAction(self.store_data) sect.setParseAction(self.add_sect) tag_sect.setParseAction(self.add_sect) sect_end.setParseAction(self.pop_sect) bnf=ZeroOrMore(input) + StringEnd().setFailAction(parse_error) bnf.ignore(pythonStyleComment) return bnf
def gramma(): ## Tokens point = Literal('.') prefix_op = Literal('.') choice_op = Literal('+') parallel = Literal("||") | Literal("<>") #ident = Word(alphas, alphanums+'_') ratename = Word(alphas.lower(),alphanums+"_") lpar = Literal('(').suppress() rpar = Literal(')').suppress() lsqpar = Literal('[').suppress() rsqpar = Literal(']').suppress() define = Literal('=') semicol = Literal(';').suppress() col = Literal(',').suppress() number = Word(nums) integer = number floatnumber = Combine( integer + Optional( point + Optional(number))) passiverate = Word('infty') | Word('T') internalrate = Word('tau') pound = Literal('#').suppress() percent = Literal('%').suppress() peparate = (ratename | floatnumber | internalrate | passiverate).setParseAction(_check_var) peparate_indef = floatnumber | internalrate | passiverate sync = Word('<').suppress() + ratename + ZeroOrMore(col + ratename) + Word('>').suppress() coop_op = (parallel | sync).setParseAction(_create_sync_set) activity = (ratename + col + peparate).setParseAction(_create_activity) procdef = (Word(alphas.upper(), alphanums+"_") + Optional(lsqpar + peparate_indef + rsqpar)).setParseAction(_create_procdef) ## RATES Definitions ratedef = (Optional(percent)+ratename + define + peparate_indef).setParseAction(assign_var) + semicol prefix = Forward() choice = Forward() coop = Forward() process = ( activity | procdef | lpar + coop + rpar ).setParseAction(_create_process) prefix << (process + ZeroOrMore(prefix_op + prefix)).setParseAction( _create_prefix) choice << (prefix + ZeroOrMore(choice_op + choice)).setParseAction(_create_choice) coop << (choice + ZeroOrMore(coop_op + coop)).setParseAction(_create_coop) rmdef = (Optional(pound) + procdef + define + coop + semicol).setParseAction(_create_definition) system_eq = Optional(pound) + coop pepa = ZeroOrMore(ratedef) + ZeroOrMore(rmdef) + system_eq.setParseAction(_create_system_equation) pepacomment = '//' + restOfLine pepa.ignore(pepacomment) return pepa
def get_language(): """Create or retrieve the parse tree for defining a sensor graph.""" global sensor_graph, statement if sensor_graph is not None: return sensor_graph _create_primitives() _create_simple_statements() _create_block_bnf() sensor_graph = ZeroOrMore(statement) + StringEnd() sensor_graph.ignore(comment) return sensor_graph
def _prepare_parser(): number = Regex(r'-?\d+') local = Regex(r'%[A-Za-z0-9._]+') glob = Regex(r'@[A-Za-z0-9._]+') meta = Regex(r'![A-Za-z0-9._]+') keywords = lambda keywords: MatchFirst( Keyword(word) for word in keywords.split()) seplist = lambda entry: delimitedList(entry) | Empty() label = local + ':' unused_def = (keywords('target declare attributes') | '!') + restOfLine type_ = Forward() void = Keyword('void') scalar_type = keywords('i1 i8 i16 i32 i64') | void types_list = seplist(type_) struct_type = '{' + types_list - '}' array_type = '[' - number - 'x' - type_ - ']' type_ << (scalar_type | local | struct_type | array_type) type_def = local + '=' + Keyword('type') - struct_type value = Forward() typed_value = type_ + value value_list = seplist(typed_value) compound_value = '{' + value_list - '}' array_value = '[' + value_list - ']' kw_value = keywords('zeroinitializer null true false') value << (number | kw_value | compound_value | array_value) linkage = Optional(keywords('private external internal common'), 'external') align = Optional(',' + Keyword('align') - number) metas = seplist(meta + meta) global_tag = keywords('global constant') initializer = Optional(value, default='undef') global_def = glob - '=' - linkage - global_tag - type_ - initializer - align - metas definition = unused_def | type_def | global_def llvm = ZeroOrMore(definition) comment = ';' + restOfLine llvm.ignore(comment) return llvm
def getkw_bnf(self): lcb = Literal("{").suppress() rcb = Literal("}").suppress() lsb = Literal("[").suppress() rsb = Literal("]").suppress() lps = Literal("(").suppress() rps = Literal(")").suppress() eql = Literal("=").suppress() dmark = Literal('$').suppress() end_sect = rcb end_data = Literal('$end').suppress() prtable = srange("[0-9a-zA-Z]") + '!$%&*+-./<>?@^_|~:' kstr = Word(prtable) ^ quotedString.setParseAction(removeQuotes) name = Word(alphas + "_", alphanums + "_") vec=lsb+delimitedList(Word(prtable) ^ Literal("\n").suppress() ^\ quotedString.setParseAction(removeQuotes))+rsb key = kstr ^ vec keyword = name + eql + kstr vector = name + eql + vec data = Combine(dmark + name) + SkipTo(end_data) + end_data data.setParseAction(self.store_data) sect = name + lcb sect.setParseAction(self.add_sect) key_sect = name + Group(lps + kstr + rps) + lcb key_sect.setParseAction(self.add_sect) vec_sect = name + Group(lps + vec + rps) + lcb vec_sect.setParseAction(self.add_vecsect) end_sect.setParseAction(self.pop_sect) keyword.setParseAction(self.store_key) vector.setParseAction(self.store_vector) section = Forward() input = section ^ data ^ keyword ^ vector sectdef = sect ^ key_sect ^ vec_sect section << sectdef + ZeroOrMore(input) + rcb bnf = ZeroOrMore(input) bnf.ignore(pythonStyleComment) return bnf
def getkw_bnf(self): sect_begin = Literal("{").suppress() sect_end = Literal("}").suppress() array_begin = Literal("[").suppress() array_end = Literal("]").suppress() arg_begin = Literal("(").suppress() arg_end = Literal(")").suppress() eql = Literal("=").suppress() dmark = Literal('$').suppress() end_data=Literal('$end').suppress() prtable = alphanums+r'!$%&*+-./<>?@^_|~' # Helper definitions kstr=Word(prtable) ^ quotedString.setParseAction(removeQuotes) name = Word(alphas+"_",alphanums+"_") vec=array_begin+delimitedList(Word(prtable) ^ \ Literal("\n").suppress() ^ \ quotedString.setParseAction(removeQuotes))+array_end sect=name+sect_begin key_sect=name+Group(arg_begin+kstr+arg_end)+sect_begin vec_sect=name+Group(arg_begin+vec+ arg_end)+sect_begin # Grammar keyword = name + eql + kstr vector = name + eql + vec data=Combine(dmark+name)+SkipTo(end_data)+end_data section=Forward() sect_def=(sect | key_sect | vec_sect) input=section | data | vector | keyword section << sect_def+ZeroOrMore(input) + sect_end # Parsing actions keyword.setParseAction(self.store_key) vector.setParseAction(self.store_vector) data.setParseAction(self.store_data) sect.setParseAction(self.add_sect) key_sect.setParseAction(self.add_sect) vec_sect.setParseAction(self.add_vecsect) sect_end.setParseAction(self.pop_sect) bnf=ZeroOrMore(input) + StringEnd().setFailAction(parse_error) bnf.ignore(pythonStyleComment) return bnf
def _grammar(self): ident = Word(alphanums + ".") semi = Literal(";").suppress() # lrb = Literal("(").suppress() # rrb = Literal(")").suppress() lcb = Literal("{").suppress() rcb = Literal("}").suppress() Value = SkipTo(semi) KeyValue = Dict(Group(ident + Value + semi)) Dictionary = Forward() Block = lcb + ZeroOrMore(Dictionary | KeyValue) + rcb Dictionary << Dict(Group(ident + Block)) ParameterFile = ZeroOrMore(Dictionary | KeyValue) ParameterFile.ignore(cStyleComment) ParameterFile.ignore(cppStyleComment) return ParameterFile
class VlogSource(VlogBase): def __repr__(self): return 'VlogSource' @syntax_root('VlogSource') def _source(self): """ source_text ::= { description } description ::= module_declaration | udp_declaration | config_declaration """ self.module = VlogPModule() self.source = ZeroOrMore(self.module.top) self.source.ignore(cStyleComment) self.source.ignore(cppStyleComment) self.source.ignore(self.compiler_directive) return self.source @syntax_tree('VlogSource', priority=10, with_name='compiler_directive') def _compiler_directive(self): """ Get rid of Compiler Directives """ # compiler directives self._compiler_directive = Combine( "`" + oneOf("define undef ifndef ifdef else endif default_nettype " "include resetall timescale unconnected_drive " "nounconnected_drive celldefine endcelldefine") + restOfLine) return self._compiler_directive
def find_procedures_headers(self): CREATE = CaselessKeyword("CREATE") OR = CaselessKeyword("OR") REPLACE = CaselessKeyword("REPLACE") FUNCTION = CaselessKeyword("FUNCTION") IN = CaselessKeyword("IN") OUT = CaselessKeyword("OUT") INOUT = CaselessKeyword("INOUT") VARIADIC = CaselessKeyword("VARIADIC") NAME = (Word(alphas, alphanums + "_."))("name") ALIAS = Word(alphas, alphanums + "_") TYPE = ( Word(alphas, alphanums + "[]_. ", ) + Suppress(Optional(Literal("(") + Word(nums) + Literal(")"))) ) PRM = ( (Optional(IN | OUT | INOUT | VARIADIC | (OUT + VARIADIC)) + Optional(ALIAS) + TYPE) | TYPE ).setParseAction(lambda res: " ".join([w.strip() for w in res])) COMMENT = "--" + restOfLine COMMA = Suppress(",") PARAMS = ZeroOrMore( PRM + Optional(COMMA) )("input") PARAMS.ignore(COMMENT) HEADER = ( CREATE + Optional(OR) + Optional(REPLACE) + FUNCTION + NAME + Suppress("(") + PARAMS + Suppress(")") ).setParseAction(lambda res: {"name": res.name, "input": res.input}) parse_header = OneOrMore(HEADER | Suppress(SkipTo(HEADER))) parse_header.ignore(COMMENT) parse_header.ignore(cStyleComment) try: headers = parse_header.parseString(self._sql) except Exception as error: print self._fpath raise error return headers
def get_parser(self): declaration = Forward() keyword = (Keyword("enum") | Keyword("case") | Keyword("struct") | Keyword("default") | Keyword("switch") | Keyword("union") | Keyword("const") | Keyword("unsigned") | Keyword("int") | Keyword("hyper") | Keyword("float") | Keyword("double") | Keyword("bool") | Keyword("typedef") | Keyword("opaque") | Keyword("string") | Keyword("void") | Keyword("program") | Keyword("version")) identifier = NotAny(keyword) + Word( alphas + alphas.upper(), alphanums + alphanums.upper() + "_", asKeyword=True) constant = Combine(Optional("-") + Word(nums)) constant.setParseAction(lambda s, l, t: [int(t[0])]) value = constant | identifier enum_body = Literal("{").suppress() + identifier + Literal( "=").suppress() + value + ZeroOrMore( Literal(",").suppress() + identifier + Literal("=").suppress() + value) + Literal("}").suppress() enum_type_spec = Literal("enum").suppress() + enum_body enum_body.setParseAction(self.parse_enum) struct_body = Literal("{").suppress() + OneOrMore( declaration + Literal(";").suppress()) + Literal("}").suppress() struct_type_spec = Literal("struct").suppress() + struct_body struct_body.setParseAction(self.parse_struct) case_stmt = Literal("case").suppress() + value + Literal( ":").suppress() + declaration + Literal(";").suppress() default_stmt = Literal("default") + Literal( ":").suppress() + declaration + Literal(";").suppress() union_body = Literal("switch").suppress() + Literal("(").suppress( ) + declaration + Literal(")").suppress() + Literal("{").suppress( ) + Group(OneOrMore(Group(case_stmt)) + Optional(Group(default_stmt))) + Literal("}").suppress() union_type_spec = Literal("union").suppress() + union_body union_body.setParseAction(self.parse_union) constant_def = Literal("const").suppress() + identifier + Literal( "=").suppress() + constant + Literal(";").suppress() constant_def.setParseAction(self.parse_const) type_spec = ((Optional(Literal("unsigned")) + Literal("int")).setParseAction(self.parse_builtin) | (Optional(Literal("unsigned")) + Literal("hyper")).setParseAction(self.parse_builtin) | Literal("float").setParseAction(self.parse_builtin) | Literal("double").setParseAction(self.parse_builtin) | Literal("bool").setParseAction(self.parse_builtin) | enum_type_spec | struct_type_spec | union_type_spec | identifier) proc_return = Literal("void") | type_spec procedure_def = proc_return + identifier + Literal("(").suppress() + ( Literal("void") | type_spec) + ZeroOrMore( Literal(",").suppress() + type_spec) + Literal(")").suppress() + Literal( "=").suppress() + constant + Literal(";").suppress() procedure_def.setParseAction(self.parse_procedure_def) version_def = Literal("version").suppress() + identifier + Literal( "{").suppress() + OneOrMore(procedure_def) + Literal("}").suppress( ) + Literal("=").suppress() + constant + Literal(";").suppress() version_def.setParseAction(self.parse_version_def) program_body = Literal("{").suppress() + Group( OneOrMore(version_def)) + Literal("}").suppress() type_def = ( (Literal("typedef") + declaration + Literal(";")) | (Literal("enum") + identifier + enum_body + Literal(";")) | (Literal("struct") + identifier + struct_body + Literal(";")) | (Literal("union") + identifier + union_body + Literal(";")) | (Literal("program") + identifier + program_body + Literal("=").suppress() + constant + Literal(";"))) type_def.setParseAction(self.parse_type_def) declaration << ( (type_spec + identifier + Literal("[") + value + Literal("]")) | (type_spec + identifier + Literal("<") + value + Literal(">")) | (type_spec + identifier) | (Literal("opaque") + identifier + Literal("[") + value + Literal("]")) | (Literal("opaque") + identifier + Literal("<") + value + Literal(">")) | (Literal("string") + identifier + Literal("<") + value + Literal(">")) | (type_spec + Literal("*") + identifier) | Literal("void")) declaration.setParseAction(self.parse_decl) definition = type_def | constant_def specification = ZeroOrMore(definition) comment = (Literal("#") + restOfLine).suppress() specification.ignore(comment) return specification
class pppCompiler: def __init__(self): self.initBNF() self.symbols = SymbolTable() def initBNF(self): indentStack = [1] encoding = Literal("<").suppress() + identifier("encoding") + Literal( ">").suppress() constdecl = Group((const + identifier + assign + value).setParseAction( self.const_action)) vardecl = Group( (type_("type_") + Optional(encoding) + identifier("name") + Optional(assign + value("value") + Optional(identifier)("unit")) ).setParseAction(self.var_action)) insertdecl = Group( (insert + dblQuotedString + LineEnd().suppress()).setParseAction( self.insert_action)) procedurecall = Group((identifier + Literal("(").suppress() + Optional( delimitedList( (identifier + Optional(assign + identifier)).setParseAction( self.named_param_action))) + Literal(")").suppress()).setParseAction( self.procedurecall_action)) condition = Group( (identifier("leftidentifier") + comparison("comparison") + (identifier("identifier") | value.setParseAction(self.value_action))).setParseAction( self.condition_action))("condition") pointer = Literal("*") + identifier rExp = Forward() #numexpression = Forward() opexpression = (identifier("operand") + (Literal(">>") | Literal("<<") | Literal("+") | Literal("*") | Literal("/") | Literal("-"))("op") + Group(rExp)("argument")).setParseAction( self.opexpression_action) rExp << ( procedurecall | opexpression | identifier("identifier") | value.setParseAction(self.value_action) | #Group( Suppress("(") + rExp + Suppress(")") ) | #Group( "+" + rExp) | #Group( "-" + rExp) | Group(Literal("not") + rExp)) rExpCondition = Group( (Optional(not_)("not_") + rExp("rExp"))).setParseAction( self.rExp_condition_action)("condition") rExp.setParseAction(self.rExp_action) assignment = ((identifier | pointer)("lval") + assign + rExp("rval")).setParseAction(self.assignment_action) addassignment = ((identifier | pointer)("lval") + (Literal("+=") | Literal("-=") | Literal("*=") | Literal("&=") | Literal("|=") | Literal(">>=") | Literal("/=") | Literal("<<="))("op") + Group(rExp)("rval")).setParseAction( self.addassignement_action) statement = Forward() statementBlock = indentedBlock(statement, indentStack).setParseAction( self.statementBlock_action) procedure_statement = Group( (Keyword("def").suppress() + identifier("funcname") + Literal("(").suppress() + Literal(")").suppress() + colon.suppress() + statementBlock).setParseAction( self.def_action)) while_statement = Group( (Keyword("while").suppress() + (condition | rExpCondition)("condition") + colon.suppress() + statementBlock("statementBlock")).setParseAction( self.while_action)) if_statement = (Keyword("if") + condition + colon + statementBlock("ifblock") + Optional( Keyword("else").suppress() + colon + statementBlock("elseblock"))).setParseAction( self.if_action) statement << (procedure_statement | while_statement | if_statement | procedurecall | assignment | addassignment) decl = constdecl | vardecl | insertdecl | Group(statement) self.program = ZeroOrMore(decl) self.program.ignore(pythonStyleComment) def assignment_action(self, text, loc, arg): logging.getLogger(__name__).debug("assignment_action {0} {1}".format( lineno(loc, text), arg)) try: code = [ "# line {0} assignment {1}".format(lineno(loc, text), line(loc, text)) ] rval_code = find_and_get(arg.rval, 'code') if rval_code is not None: code += arg.rval.code elif arg.rval == "*P": code.append(" LDWI") elif 'identifier' in arg: self.symbols.getVar(arg.identifier) code.append(" LDWR {0}".format(arg.identifier)) if arg.lval == "*P": code.append(" STWI") elif arg.lval != "W": symbol = self.symbols.getVar(arg.lval) code.append(" STWR {0}".format(symbol.name)) if 'code' in arg: arg['code'].extend(code) else: arg['code'] = code except Exception as e: raise CompileException(text, loc, str(e), self) return arg def addassignement_action(self, text, loc, arg): logging.getLogger(__name__).debug( "addassignement_action {0} {1}".format(lineno(loc, text), arg)) try: code = [ "# line {0}: add_assignment: {1}".format( lineno(loc, text), line(loc, text)) ] if arg.rval[0] == '1' and arg.op in ['+=', '-=']: self.symbols.getVar(arg.lval) if arg.op == "+=": code.append(" INC {0}".format(arg.lval)) else: code.append(" DEC {0}".format(arg.lval)) else: if 'code' in arg.rval: code += arg.rval.code self.symbols.getVar(arg.lval) if arg.op == "-=": raise CompileException( "-= with expression needs to be fixed in the compiler" ) code.append(" {0} {1}".format(opassignmentLookup[arg.op], arg.lval)) elif 'identifier' in arg.rval: self.symbols.getVar(arg.rval.identifier) code.append(" LDWR {0}".format(arg.lval)) self.symbols.getVar(arg.lval) code.append(" {0} {1}".format(opassignmentLookup[arg.op], arg.rval.identifier)) code.append(" STWR {0}".format(arg.lval)) arg['code'] = code except Exception as e: raise CompileException(text, loc, str(e), self) return arg def condition_action(self, text, loc, arg): logging.getLogger(__name__).debug("condition_action {0} {1}".format( lineno(loc, text), arg)) try: code = [ "# line {0} condition {1}".format(lineno(loc, text), line(loc, text)) ] if arg.leftidentifier != "W": self.symbols.getVar(arg.leftidentifier) code.append(' LDWR {0}'.format(arg.leftidentifier)) if arg.identifier == 'NULL' and arg.comparison in jmpNullCommands: arg['jmpcmd'] = jmpNullCommands[arg.comparison] else: code.append(' {0} {1}'.format( comparisonCommands[arg.comparison], arg.identifier)) arg["code"] = code except Exception as e: raise CompileException(text, loc, str(e), self) return arg def rExp_condition_action(self, text, loc, arg): logging.getLogger(__name__).debug( "rExp_condition_action {0} {1}".format(lineno(loc, text), arg)) try: code = [ "# line {0} rExp_condition {1}".format(lineno(loc, text), line(loc, text)) ] condition_code = arg.condition.rExp['code'] if isinstance(condition_code, str): if 'not_' in arg['condition']: code += [" CMPEQUAL NULL"] else: code += [" CMPNOTEQUAL NULL"] arg['code'] = code else: if 'not_' in arg['condition']: arg['code'] = { False: condition_code[True], True: condition_code[False] } else: arg['code'] = condition_code except Exception as e: raise CompileException(text, loc, str(e), self) return arg def named_param_action(self, text, loc, arg): if len(arg) == 2: arg[arg[0]] = arg[1] return arg def value_action(self, text, loc, arg): if arg[0][0:2] == '0x': value = int(arg[0], 16) else: value = int(arg[0]) arg["identifier"] = self.symbols.getInlineParameter("inlinevar", value) return arg def opexpression_action(self, text, loc, arg): try: logging.getLogger(__name__).debug( "opexpression_action {0} {1}".format(lineno(loc, text), arg)) code = [ "# line {0}: shiftexpression {1}".format( lineno(loc, text), line(loc, text)), " LDWR {0}".format(arg.operand), " {0} {1}".format(shiftLookup[arg.op], arg.argument.identifier) ] arg['code'] = code logging.getLogger(__name__).debug( "shiftexpression generated code {0}".format(code)) except Exception as e: raise CompileException(text, loc, str(e), self) return arg def procedurecall_action(self, text, loc, arg): try: logging.getLogger(__name__).debug( "procedurecall_action {0} {1}".format(lineno(loc, text), arg)) procedure = self.symbols.getProcedure(arg[0]) code = [ "# line {0}: procedurecall {1}".format(lineno(loc, text), line(loc, text)) ] opcode = procedure.codegen(self.symbols, arg=arg.asList(), kwarg=arg.asDict()) if isinstance(opcode, list): code += opcode else: code = opcode arg['code'] = code logging.getLogger(__name__).debug( "procedurecall generated code {0}".format(code)) except Exception as e: raise CompileException(text, loc, str(e), self) return arg def rExp_action(self, text, loc, arg): logging.getLogger(__name__).debug("rExp_action {0} {1}".format( lineno(loc, text), arg)) pass def if_action(self, text, loc, arg): logging.getLogger(__name__).debug("if_action {0} {1}".format( lineno(loc, text), arg)) try: block0 = [ "# line {0} if statement {1}".format(lineno(loc, text), line(loc, text)) ] if isinstance(arg.condition.code, list): block0 += arg.condition.code JMPCMD = arg.condition.get('jmpcmd', {False: "JMPNCMP"})[False] else: JMPCMD = arg.condition.code[True] if 'elseblock' in arg: block1 = arg.ifblock.ifblock.code block2 = arg.elseblock.elseblock[ 'code'] if 'elseblock' in arg.elseblock else arg.elseblock[ 'code'] else: block1 = arg.ifblock.ifblock['code'] block2 = None arg['code'] = [ IfGenerator(self.symbols, JMPCMD, block0, block1, block2) ] except Exception as e: raise CompileException(text, loc, str(e), self) return arg def while_action(self, text, loc, arg): logging.getLogger(__name__).debug("while_action {0} {1}".format( lineno(loc, text), arg)) try: block0 = [ "# line {0} while_statement {1}".format( lineno(loc, text), line(loc, text)) ] if 'code' in arg.condition: if isinstance(arg.condition.code, list): block1 = arg.condition.code JMPCMD = arg.condition.get('jmpcmd', {False: "JMPNCMP"})[False] else: JMPCMD = arg.condition.code[True] block1 = [] elif 'rExp' in arg.condition and 'code' in arg.condition.rExp: if isinstance(arg.condition.rExp.code, list): block1 = arg.condition.rExp.code JMPCMD = arg.condition.rExp.get('jmpcmd', "JMPNCMP") else: JMPCMD = arg.condition.rExp.code[True] block1 = [] block2 = arg.statementBlock.statementBlock['code'] arg['code'] = [ WhileGenerator(self.symbols, JMPCMD, block0, block1, block2) ] logging.getLogger(__name__).debug("while_action generated code ") except Exception as e: raise CompileException(text, loc, str(e), self) return arg def statementBlock_action(self, text, loc, arg): logging.getLogger(__name__).debug( "statementBlock_action {0} {1} {2}".format(lineno(loc, text), arg.funcname, arg)) try: code = list() for command in arg[0]: if 'code' in command: code += command['code'] elif 'code' in command[0]: code += command[0]['code'] arg[0]['code'] = code logging.getLogger(__name__).debug( "statementBlock generated code {0}".format(code)) except Exception as e: raise CompileException(text, loc, str(e), self) return arg def def_action(self, text, loc, arg): logging.getLogger(__name__).debug("def_action {0} {1} {2}".format( lineno(loc, text), arg.funcname, arg)) try: name = arg[0] self.symbols.checkAvailable(name) self.symbols[name] = FunctionSymbol(name, arg[1]['code']) except Exception as e: raise CompileException(text, loc, str(e), self) def const_action(self, text, loc, arg): try: name, value = arg logging.getLogger(__name__).debug( "const_action {0} {1} {2} {3}".format(self.currentFile, lineno(loc, text), name, value)) self.symbols[name] = ConstSymbol(name, value) except Exception as e: raise CompileException(text, loc, str(e), self) def var_action(self, text, loc, arg): logging.getLogger(__name__).debug( "var_action {0} {1} {2} {3} {4} {5} {6}".format( self.currentFile, lineno(loc, text), arg["type_"], arg.get("encoding"), arg["name"], arg.get("value"), arg.get("unit"))) try: type_ = arg["type_"] if arg["type_"] != "var" else None self.symbols[arg["name"]] = VarSymbol(type_=type_, name=arg["name"], value=arg.get("value"), encoding=arg.get("encoding"), unit=arg.get("unit")) except Exception as e: raise CompileException(text, loc, str(e), self) def insert_action(self, text, loc, arg): try: oldfile = self.currentFile myprogram = self.program.copy() self.currentFile = arg[0][1:-1] result = myprogram.parseFile(self.currentFile) self.currentFile = oldfile except Exception as e: raise CompileException(text, loc, str(e), self) return result def compileFile(self, filename): self.currentFile = filename result = self.program.parseFile(self.currentFile, parseAll=True) allcode = list() for element in result: if not isinstance(element, str) and 'code' in element: allcode += element['code'] elif not isinstance(element[0], str) and 'code' in element[0]: allcode += element[0]['code'] header = self.createHeader() codetext = "\n".join(header + allcode) return codetext def compileString(self, programText): self.programText = programText self.currentFile = "Memory" result = self.program.parseString(self.programText, parseAll=True) allcode = list() for element in result: if not isinstance(element, str) and 'code' in element: allcode += element['code'] elif not isinstance(element[0], str) and 'code' in element[0]: allcode += element[0]['code'] header = self.createHeader() codetext = """# autogenerated # DO NOT EDIT DIRECTLY # The file will be overwritten by the compiler # """ codetext += "\n".join(header + list(generate(allcode))) self.reverseLineLookup = self.generateReverseLineLookup(codetext) return codetext def generateReverseLineLookup(self, codetext): lookup = dict() sourceline = None for codeline, line in enumerate(codetext.splitlines()): m = re.search('^\# line (\d+).*$', line) if m: sourceline = int(m.group(1)) else: lookup[codeline + 1] = sourceline return lookup def createHeader(self): header = ["# const values"] for constval in self.symbols.getAllConst(): header.append("const {0} {1}".format(constval.name, constval.value)) header.append("# variables ") for var in self.symbols.getAllVar(): if var.type_ == "masked_shutter": header.append("var {0} {1}, {2}".format( var.name + "_mask", var.value if var.value is not None else 0, "mask")) header.append("var {0} {1}, {2}".format( var.name, var.value if var.value is not None else 0, "shutter {0}_mask".format(var.name))) else: optionals = [ s if s is not None else "" for s in list_rtrim([var.type_, var.unit, var.encoding]) ] varline = "var {0} {1}".format( var.name, var.value if var.value is not None else 0) if len(optionals) > 0: varline += ", " + ", ".join(optionals) header.append(varline) header.append("# inline variables") # for value, name in self.symbols.inlineParameterValues.items(): # header.append("var {0} {1}".format(name, value)) header.append("# end header") header.append("") return header
identifier.setResultsName("tablename") + fromToken + identifier.setResultsName("filename") #orderByClause = orderByToken + selectStatement << (selectToken + columnNameList.setResultsName("columns") + fromToken + identifier.setResultsName("tablename") + Optional(whereClause) + Optional(orderByClause) + Optional(limitClause)) BQLStatement = (selectStatement | createBtableStatement) + Optional(';') BQL = ZeroOrMore(BQLStatement) ## allows comments dashComment = "--" + restOfLine BQL.ignore(dashComment) def test(str): print str, "->" try: tokens = BQL.parseString(str) print "tokens = ", tokens print "tokens.tablename =", tokens.tablename print "tokens.filename =", tokens.filename #print "tokens.where =", tokens.where except ParseException, err: print " " * err.loc + "^\n" + err.msg print err print
class PulseProgram: """ Encapsulates a PulseProgrammer Program loadSource( filename ) loads the contents of the file The code is compiled in the following steps parse() generates self.code toBytecode() generates self.bytecode toBinary() generates self.binarycode the procedure updateVariables( dictionary ) updates variable values in the bytecode """ def __init__(self): self.variabledict = collections.OrderedDict( ) # keeps information on all variables to easily change them later self.labeldict = dict() # keep information on all labels self.source = collections.OrderedDict( ) # dictionary of source code files (stored as strings) self.code = [] # this is a list of lines self.bytecode = [] # list of op, argument tuples self.binarycode = bytearray() # binarycode to be uploaded self._exitcodes = dict( ) # generate a reverse dictionary of variables of type exitcode self.constDict = dict() class Board: channelLimit = 1 halfClockLimit = 500000000 self.adIndexList = [(x, 0) for x in range(6)] self.adBoards = [Board()] * 6 self.timestep = Q(20.0, 'ns') self.initBNF() def initBNF(self): constdecl = (CONST + NAME + VALUE).setParseAction(self.const_action) vardecl = (VAR + NAME + VALUE + Optional(COMMA + Regex("[^#\n]*"))).setParseAction( self.var_action) insertdecl = (INSERT + dblQuotedString + LineEnd().suppress()).setParseAction(self.insert_action) LABEL = IDENTIFIER + COLON COMMANDEXP = (IDENTIFIER.setWhitespaceChars(" \t") + Regex("[^#\n]*").setWhitespaceChars(" \t") + LineEnd().suppress()) COMMAND = COMMANDEXP.setParseAction(self.command_action) LABELEDCOMMAND = (LABEL + COMMANDEXP).setParseAction( self.label_command_action) decl = constdecl | vardecl | insertdecl | LABELEDCOMMAND | COMMAND self.program = ZeroOrMore(decl) self.program.ignore(pythonStyleComment) def const_action(self, text, loc, arg): """ add the const to the self.constDict dictionary """ logger = logging.getLogger(__name__) logger.debug("{0}:{1} const {2}".format(self.currentFile, lineno(loc, text), arg)) label, value = arg if label in self.constDict: logger.error( "Error parsing const in file '{0}': attempted to redefine'{1}' to '{2}' from '{3}'" .format(self.currentFile, label, value, self.constDict[label])) raise ppexception("Redefining variable", self.currentFile, lineno, label) else: self.constDict[label] = int(value) def var_action(self, text, loc, arg): print("var_action", self.currentFile, lineno(loc, text), arg[0:2], arg[2].split(",") if len(arg) > 2 else "") """ add a variable to the self.variablesdict """ logger = logging.getLogger(__name__) logger.debug("{0}:{1} Variable {2}".format(self.currentFile, lineno(loc, text), arg)) var = Variable() label, data = arg[:2] fields = arg[2].split(",") if len(arg) > 2 else [None] * 3 fields += [None] * (3 - len(fields)) var.type, unit, var.encoding = [ x if x is None or '' else x.strip() for x in fields ] var.name = label var.origin = self.currentFile var.enabled = True if var.encoding not in encodings: raise ppexception( "unknown encoding {0} in file '{1}':{2}".format( var.encoding, self.currentFile, lineno(loc, text)), self.currentFile, lineno, var.encoding) try: data = str(eval(data, globals(), self.defines)) except Exception: logger.exception( "Evaluation error in file '{0}' on line: '{1}'".format( self.currentFile, data)) if unit is not None: var.value = Q(float(data), unit) data = self.convertParameter(var.value, var.encoding) else: var.value = Q(float(data)) data = int(round(float(data))) if label in self.defines: logger.error( "Error in file '%s': attempted to reassign '%s' to '%s' (from prev. value of '%s') in a var statement." % (self.currentFile, label, data, self.defines[label])) raise ppexception("variable redifinition", self.currentFile, lineno, label) else: self.defines[ label] = label # add the variable to the dictionary of definitions to prevent identifiers and variables from having the same name # however, we do not want it replaced with a number but keep the name for the last stage of compilation pass var.data = data self.variabledict.update({label: var}) if var.type == "exitcode": self._exitcodes[data & 0x0000ffff] = var def command_action(self, text, loc, arg): print("command_action", self.currentFile, lineno(loc, text), arg[0:1], arg[1].split(",") if len(arg) > 1 else "") def label_command_action(self, text, loc, arg): print("label_command_action", self.currentFile, lineno(loc, text), arg[0:2], arg[2].split(",") if len(arg) > 2 else "") def addLabel(self, label, address, sourcename, lineno): if label is not None: self.labeldict[label] = address def insert_action(self, text, loc, arg): oldfile = self.currentFile print("insert_action", lineno(loc, text), arg) myprogram = self.program.copy() self.currentFile = arg[0][1:-1] result = myprogram.parseFile(self.currentFile) self.currentFile = oldfile print(result) return result def assembleFile(self, filename): self.currentFile = filename result = self.program.parseFile(self.currentFile) return result def setHardware(self, adIndexList, adBoards, timestep): self.adIndexList = adIndexList self.adBoards = adBoards self.timestep = timestep assert self.timestep.has_dimension('s') def saveSource(self): for name, text in self.source.items(): with open(os.path.join(self.pp_dir, name), 'w') as f: f.write(text) def loadSource(self, pp_file): """ Load the source pp_file #include files are loaded recursively all code lines are added to self.sourcelines for each source file the contents are added to the dictionary self.source """ self.source.clear() self.pp_dir, self.pp_filename = os.path.split(pp_file) self.sourcelines = [] self.insertSource(self.pp_filename) self.compileCode() def updateVariables(self, variables): """ update the variable values in the bytecode """ logger = logging.getLogger(__name__) for name, value in variables.items(): if name in self.variabledict: var = self.variabledict[name] address = var.address var.value = value logger.debug( "updateVariables {0} at address 0x{2:x} value {1}, 0x{3:x}" .format(name, value, address, int(var.data))) var.data = self.convertParameter(value, var.encoding) self.bytecode[address] = (self.bytecode[address][0], var.data) self.variabledict[name] = var else: logger.error( "variable {0} not found in dictionary.".format(name)) return self.bytecode def variables(self): mydict = dict() for name, var in self.variabledict.items(): mydict.update({name: var.value}) return mydict def variable(self, variablename): return self.variabledict.get(variablename).value def variableUpdateCode(self, variablename, value): """returns the code to update the variable directly on the fpga consists of variablelocation and variablevalue """ var = self.variabledict[variablename] data = self.convertParameter(value, var.encoding) return bytearray(struct.pack('II', (var.address, data))) def flattenList(self, l): return [item for sublist in l for item in sublist] def variableScanCode(self, variablename, values): var = self.variabledict[variablename] # [item for sublist in l for item in sublist] idiom for flattening of list return self.flattenList([(var.address, self.convertParameter(x, var.encoding)) for x in values]) def loadFromMemory(self): """Similar to loadSource only this routine loads from self.source """ self.sourcelines = [] self._exitcodes = dict() self.insertSource(self.pp_filename) self.compileCode() def toBinary(self): """ convert bytecode to binary """ logger = logging.getLogger(__name__) self.binarycode = bytearray() for wordno, (op, arg) in enumerate(self.bytecode): logger.debug("{0} {1} {2} {3}".format( hex(wordno), hex(int(op)), hex(int(arg)), hex(int((int(op) << 24) + int(arg))))) self.binarycode += struct.pack('I', int((op << 24) + arg)) return self.binarycode def currentVariablesText(self): lines = list() for name, var in iter(sorted(self.variabledict.items())): lines.append("{0} {1}".format(name, var.value)) return '\n'.join(lines) # routines below here should not be needed by the user insertPattern = re.compile('#insert\s+([\w.-_]+)', re.IGNORECASE) codelinePattern = re.compile('(#define|\s*[^#\s]+)', re.IGNORECASE) def insertSource(self, pp_file): """ read a source file pp_file calls itself recursively to for #insert adds the contents of this file to the dictionary self.source """ logger = logging.getLogger(__name__) if pp_file not in self.source: with open(os.path.join(self.pp_dir, pp_file)) as f: self.source[pp_file] = ''.join(f.readlines()) sourcecode = self.source[pp_file] for line, text in enumerate(sourcecode.splitlines()): m = self.insertPattern.match(text) if m: filename = m.group(1) logger.info("inserting code from {0}".format(filename)) self.insertSource(filename) else: if self.codelinePattern.match(text): self.sourcelines.append((text, line + 1, pp_file)) labelPattern = re.compile('(\w+):\s+([^#\n\r]*)') opPattern = re.compile('\s*(\w+)(?:\s+([^#\n\r]*)){0,1}', re.IGNORECASE) varPattern = re.compile( 'var\s+(\w+)\s+([^#,\n\r]+)(?:,([^#,\n\r]+)){0,1}(?:,([^#,\n\r]+)){0,1}(?:,([^#,\n\r]+)){0,1}(?:#([^\n\r]+)){0,1}' ) # def parse(self): """ parse the code """ logger = logging.getLogger(__name__) self.code = [] self.variabledict = collections.OrderedDict() self.defines = dict() addr_offset = 0 for text, lineno, sourcename in self.sourcelines: m = self.varPattern.match(text) if m: self.addVariable(m, lineno, sourcename) else: m = self.definePattern.match(text) if m: self.addDefine(m, lineno, sourcename) else: # extract any JMP label, if present m = self.labelPattern.match(text) if m: label, text = m.groups( ) #so the operation after ":" still gets parsed CWC 08162012 else: label = None #The label for non-jump label line is NONE CWC 08172012 # search OPS list for a match to the current line m = self.opPattern.match(text) if m: op, args = m.groups() op = op.upper() # split and remove whitespace arglist = [0] if args is None else [ 0 if x is None else x.strip() for x in args.split(',') ] #substitute the defined variable directly with the corresponding value CWC 08172012 arglist = [ self.defines[x] if x in self.defines else x for x in arglist ] #check for dds commands so CHAN commands can be inserted if (op[:3] == 'DDS'): try: board = self.adIndexList[int(arglist[0])][0] except ValueError: raise ppexception( "DDS argument does not resolve to integer", sourcename, lineno, arglist[0]) chan = self.adIndexList[int(arglist[0])][1] if (self.adBoards[board].channelLimit != 1): #boards with more than one channel require an extra channel selection command chanData = self.adBoards[board].addCMD(chan) chanData += (int(board) << 16) self.code.append( (len(self.code) + addr_offset, 'DDSCHN', chanData, label, sourcename, lineno)) data = arglist if len(arglist) > 1 else arglist[0] self.addLabel(label, len(self.code), sourcename, lineno) self.code.append((len(self.code) + addr_offset, op, data, label, sourcename, lineno)) else: logger.error( "Error processing line {2}: '{0}' in file '{1}' (unknown opcode?)" .format(text, sourcename, lineno)) raise ppexception( "Error processing line {2}: '{0}' in file '{1}' (unknown opcode?)" .format(text, sourcename, lineno), sourcename, lineno, text) self.appendVariableCode() return self.code def appendVariableCode(self): """ append all variables to the instruction part of the code """ for var in list(self.variabledict.values()): address = len(self.code) self.code.append((address, 'NOP', var.data if var.enabled else 0, None, var.origin, 0)) var.address = address def addVariable(self, m, lineno, sourcename): """ add a variable to the self.variablesdict """ logger = logging.getLogger(__name__) logger.debug("Variable {0} {1} {2}".format(m.groups(), lineno, sourcename)) var = Variable() label, data, var.type, unit, var.encoding, var.comment = [ x if x is None else x.strip() for x in m.groups() ] var.name = label var.origin = sourcename var.enabled = True if var.encoding not in encodings: raise ppexception( "unknown encoding {0} in file '{1}':{2}".format( var.encoding, sourcename, lineno), sourcename, lineno, var.encoding) try: data = str(eval(data, globals(), self.defines)) except Exception: logger.exception( "Evaluation error in file '{0}' on line: '{1}'".format( sourcename, data)) if unit is not None: var.value = Q(float(data), unit) data = self.convertParameter(var.value, var.encoding) else: var.value = Q(float(data)) # var.value.output_prec(0) # without dimension the parameter has to be int. Thus, we do not want decimal places :) data = int(round(float(data))) if label in self.defines: logger.error( "Error in file '%s': attempted to reassign '%s' to '%s' (from prev. value of '%s') in a var statement." % (sourcename, label, data, self.defines[label])) raise ppexception("variable redifinition", sourcename, lineno, label) else: self.defines[ label] = label # add the variable to the dictionary of definitions to prevent identifiers and variables from having the same name # however, we do not want it replaced with a number but keep the name for the last stage of compilation pass var.data = data self.variabledict.update({label: var}) if var.type == "exitcode": self._exitcodes[data & 0x0000ffff] = var # code is (address, operation, data, label or variablename, currentfile) def toBytecode(self): """ generate bytecode from code """ logger = logging.getLogger(__name__) logger.debug("\nCode ---> ByteCode:") self.bytecode = [] for line in self.code: logger.debug("{0}: {1}".format(hex(line[0]), line[1:])) bytedata = 0 if line[1] not in OPS: raise ppexception("Unknown command {0}".format(line[1]), line[4], line[5], line[1]) byteop = OPS[line[1]] try: data = line[2] #attempt to locate commands with constant data if (data == ''): #found empty data bytedata = 0 elif isinstance(data, (int, int)): bytedata = data elif isinstance(data, float): bytedata = int(data) elif isinstance( data, str ): # now we are dealing with a variable and need its address bytedata = self.variabledict[line[2]].address if line[ 2] in self.variabledict else self.labeldict[line[2]] elif isinstance( data, list ): # list is what we have for DDS, will have 8bit channel and 16bit address channel, data = line[2] if isinstance(data, str): data = self.variabledict[data].address bytedata = ( (int(channel) & 0xf) << 16) | (int(data) & 0x0fff) except KeyError: logger.error( "Error assembling bytecode from file '{0}': Unknown variable: '{1}'. \n" .format(line[4], data)) raise ppexception( "{0}: Unknown variable {1}".format(line[4], data), line[4], line[5], data) self.bytecode.append((byteop, bytedata)) logger.debug("---> {0} {1}".format(hex(byteop), hex(bytedata))) return self.bytecode def convertParameter(self, mag, encoding=None): """ convert a dimensioned parameter to the binary value expected by the hardware. The conversion is determined by the variable encoding """ if is_Q(mag): if mag.dimensionality == Dimensions.time: result = int(round(mag / self.timestep)) else: step, unit, _, mask = encodings[encoding] result = int(round(mag.m_as(unit) / step)) & mask else: if encoding: step, unit, _, mask = encodings[encoding] result = int(round(mag / step)) & mask else: result = mag return result def compileCode(self): self.parse() self.toBytecode() def exitcode(self, code): if code in self._exitcodes: var = self._exitcodes[code] if var.comment: return var.comment else: return var.name else: return "Exitcode {0} Not found".format(code)
def SPICE_BNF(): global bnf if not bnf: # punctuation colon = Literal(":").suppress() lbrace = Literal("{").suppress() rbrace = Literal("}").suppress() lbrack = Literal("[").suppress() rbrack = Literal("]").suppress() lparen = Literal("(").suppress() rparen = Literal(")").suppress() equals = Literal("=").suppress() comma = Literal(",").suppress() semi = Literal(";").suppress() # primitive types int8_ = Keyword("int8").setParseAction(replaceWith(ptypes.int8)) uint8_ = Keyword("uint8").setParseAction(replaceWith(ptypes.uint8)) int16_ = Keyword("int16").setParseAction(replaceWith(ptypes.int16)) uint16_ = Keyword("uint16").setParseAction(replaceWith(ptypes.uint16)) int32_ = Keyword("int32").setParseAction(replaceWith(ptypes.int32)) uint32_ = Keyword("uint32").setParseAction(replaceWith(ptypes.uint32)) int64_ = Keyword("int64").setParseAction(replaceWith(ptypes.int64)) uint64_ = Keyword("uint64").setParseAction(replaceWith(ptypes.uint64)) # keywords channel_ = Keyword("channel") enum32_ = Keyword("enum32").setParseAction(replaceWith(32)) enum16_ = Keyword("enum16").setParseAction(replaceWith(16)) enum8_ = Keyword("enum8").setParseAction(replaceWith(8)) flags32_ = Keyword("flags32").setParseAction(replaceWith(32)) flags16_ = Keyword("flags16").setParseAction(replaceWith(16)) flags8_ = Keyword("flags8").setParseAction(replaceWith(8)) channel_ = Keyword("channel") server_ = Keyword("server") client_ = Keyword("client") protocol_ = Keyword("protocol") typedef_ = Keyword("typedef") struct_ = Keyword("struct") message_ = Keyword("message") image_size_ = Keyword("image_size") bytes_ = Keyword("bytes") cstring_ = Keyword("cstring") switch_ = Keyword("switch") default_ = Keyword("default") case_ = Keyword("case") identifier = Word(alphas, alphanums + "_") enumname = Word(alphanums + "_") integer = ( Combine(CaselessLiteral("0x") + Word(nums + "abcdefABCDEF")) | Word(nums + "+-", nums)).setName("int").setParseAction(cvtInt) typename = identifier.copy().setParseAction( lambda toks: ptypes.TypeRef(str(toks[0]))) # This is just normal "types", i.e. not channels or messages typeSpec = Forward() attributeValue = integer ^ identifier attribute = Group( Combine("@" + identifier) + Optional(lparen + delimitedList(attributeValue) + rparen)) attributes = Group(ZeroOrMore(attribute)) arraySizeSpecImage = Group(image_size_ + lparen + integer + comma + identifier + comma + identifier + rparen) arraySizeSpecBytes = Group(bytes_ + lparen + identifier + comma + identifier + rparen) arraySizeSpecCString = Group(cstring_ + lparen + rparen) arraySizeSpec = lbrack + Optional( identifier ^ integer ^ arraySizeSpecImage ^ arraySizeSpecBytes ^ arraySizeSpecCString, default="") + rbrack variableDef = Group(typeSpec + Optional("*", default=None) + identifier + Optional(arraySizeSpec, default=None) + attributes - semi) \ .setParseAction(parseVariableDef) switchCase = Group(Group(OneOrMore(default_.setParseAction(replaceWith(None)) + colon | Group(case_.suppress() + Optional("!", default="") + identifier) + colon)) + variableDef) \ .setParseAction(lambda toks: ptypes.SwitchCase(toks[0][0], toks[0][1])) switchBody = Group(switch_ + lparen + delimitedList(identifier,delim='.', combine=True) + rparen + lbrace + Group(OneOrMore(switchCase)) + rbrace + identifier + attributes - semi) \ .setParseAction(lambda toks: ptypes.Switch(toks[0][1], toks[0][2], toks[0][3], toks[0][4])) messageBody = structBody = Group(lbrace + ZeroOrMore(variableDef | switchBody) + rbrace) structSpec = Group(struct_ + identifier + structBody + attributes).setParseAction( lambda toks: ptypes.StructType( toks[0][1], toks[0][2], toks[0][3])) # have to use longest match for type, in case a user-defined type name starts with a keyword type, like "channel_type" typeSpec << (structSpec ^ int8_ ^ uint8_ ^ int16_ ^ uint16_ ^ int32_ ^ uint32_ ^ int64_ ^ uint64_ ^ typename).setName("type") flagsBody = enumBody = Group( lbrace + delimitedList(Group(enumname + Optional(equals + integer))) + Optional(comma) + rbrace) messageSpec = Group(message_ + messageBody + attributes ).setParseAction(lambda toks: ptypes.MessageType( None, toks[0][1], toks[0][2])) | typename channelParent = Optional(colon + typename, default=None) channelMessage = Group(messageSpec + identifier + Optional(equals + integer, default=None) + semi) \ .setParseAction(lambda toks: ptypes.ChannelMember(toks[0][1], toks[0][0], toks[0][2])) channelBody = channelParent + Group(lbrace + ZeroOrMore( server_ + colon | client_ + colon | channelMessage) + rbrace) enum_ = (enum32_ | enum16_ | enum8_) flags_ = (flags32_ | flags16_ | flags8_) enumDef = Group(enum_ + identifier + enumBody + attributes - semi).setParseAction(lambda toks: ptypes.EnumType( toks[0][0], toks[0][1], toks[0][2], toks[0][3])) flagsDef = Group(flags_ + identifier + flagsBody + attributes - semi).setParseAction(lambda toks: ptypes.FlagsType( toks[0][0], toks[0][1], toks[0][2], toks[0][3])) messageDef = Group(message_ + identifier + messageBody + attributes - semi).setParseAction( lambda toks: ptypes.MessageType( toks[0][1], toks[0][2], toks[0][3])) channelDef = Group(channel_ + identifier + channelBody - semi).setParseAction( lambda toks: ptypes.ChannelType( toks[0][1], toks[0][2], toks[0][3])) structDef = Group(struct_ + identifier + structBody + attributes - semi).setParseAction(lambda toks: ptypes.StructType( toks[0][1], toks[0][2], toks[0][3])) typedefDef = Group(typedef_ + identifier + typeSpec + attributes - semi).setParseAction(lambda toks: ptypes.TypeAlias( toks[0][1], toks[0][2], toks[0][3])) definitions = typedefDef | structDef | enumDef | flagsDef | messageDef | channelDef protocolChannel = Group(typename + identifier + Optional(equals + integer, default=None) + semi) \ .setParseAction(lambda toks: ptypes.ProtocolMember(toks[0][1], toks[0][0], toks[0][2])) protocolDef = Group(protocol_ + identifier + Group(lbrace + ZeroOrMore(protocolChannel) + rbrace) + semi) \ .setParseAction(lambda toks: ptypes.ProtocolType(toks[0][1], toks[0][2])) bnf = ZeroOrMore(definitions) + protocolDef + StringEnd() singleLineComment = "//" + restOfLine bnf.ignore(singleLineComment) bnf.ignore(cStyleComment) return bnf
class SFZParser(object): def __init__(self, path, text, state=None): self.path = path self.base_path = os.path.dirname(path) self.text = text self.state = state opcode_name = Word(alphanums + '_') value = Regex(r'.*?(?=\s*(([a-zA-Z0-9_]+=)|//|<[a-z]|$))', re.MULTILINE) opcode = locatedExpr(opcode_name) + Literal('=').suppress() + value opcode.setParseAction(self.handle_opcode) section_name = Literal('<').suppress() + Word(alphas) + Literal('>').suppress() section = section_name section.setParseAction(self.handle_section) include = Literal('#include').suppress() + locatedExpr(QuotedString('"')) include.setParseAction(self.handle_include) statement = (section ^ opcode ^ include) self.sfz_file = ZeroOrMore(statement) + stringEnd comment = Literal('//') + restOfLine self.sfz_file.ignore(comment) def handle_include(self, s, loc, toks): path = os.path.join(self.base_path, normalize_path(toks[0].value)) try: with open(path) as fp: f = fp.read() except IOError as exc: raise IncludeException( s, loc=toks[0].locn_start, msg=str(exc)) subparser = SFZParser(path, f, self.state) subparser.sfz_file.parseString(f) def handle_section(self, s, loc, toks): name = toks[0] if name == 'region': section = Region(self.state.instr, name, group=self.state.current_group, control=self.state.current_control) self.state.instr.regions.append(section) elif name == 'group': section = Section(self.state.instr, name) self.state.current_group = section elif name == 'control': section = Section(self.state.instr, name) self.state.current_control = section else: raise InvalidSectionException( s, loc, "Invalid section name '%s'" % name) self.state.current_section = section def handle_opcode(self, s, loc, toks): loc = toks[0].locn_start name = toks[0].value try: opdef = opmap[name] except KeyError: raise UnknownOpCodeException( s, loc=loc, msg="Unknown opcode '%s'" % key) try: value = opdef.parse_value(toks[1]) except ValueError as exc: raise InvalidValueException( s, loc=loc, msg="Invalid value for opcode '%s': %s" % (key, str(exc))) self.state.current_section._opcodes[name] = value self.state.current_section._opcode_locs[name] = (s, loc) def parse(self): self.state = ParserState() self.state.instr = Instrument(os.path.abspath(self.path)) self.sfz_file.parseString(self.text) for region in self.state.instr.regions: if not os.path.isfile(region.sample): s, loc = region.get_opcode_loc('sample') raise SampleMissingException( s, loc, "Missing sample '%s'" % region.sample) return self.state.instr
optionDirective = OPTION_ - IDENT("optionName") + EQ + quotedString( "optionValue") + SEMI #annotateDef = AT + (IDENT + EQ + (quotedString | STRQ3)).setParseAction(addToAnnotationDict) annotateDef = AT + (IDENT + EQ + ANNOTSTR).setParseAction(addToAnnotationDict) #topLevelStatement = Group(structDefn | structExtension | enumDefn | serviceDefn | importDirective | optionDirective) ##topLevelStatement = Group(structDefn | structExtension | enumDefn | importDirective | optionDirective) topLevelStatement = Group(annotateDef | structDefn | enumDefn | importDirective | optionDirective) ##parser = Optional(packageDirective) + ZeroOrMore(topLevelStatement) #parser = Group(CMNT) + ZeroOrMore(topLevelStatement) parser = ZeroOrMore(topLevelStatement) parser.ignore(comment) test1 = """ @name = "Test BBX" @version = "0.1-4" @doc_title = "BBX document" @doc_header = "BBX document heading generation Testing" @doc_intro = '''This is the definiton of the message protocol used for bla-bla-bla''' @c_includes = ''' #include <stdio.h> #include "comms.h" ''' @c_code = '''
ddlConstraint = Or([ ddlWord("CONSTRAINT"), ddlWord("PRIMARY"), ddlWord("FOREIGN"), ddlWord("KEY"), ddlWord("INDEX"), ddlWord("UNIQUE"), ]) ddlColumn = Group(Optional(ddlConstraint).setResultsName("isConstraint") + OneOrMore(MatchFirst([ddlNotNull, ddlAutoValue, ddlDefaultValue, ddlTerm, ddlNum, ddlColumnComment, ddlString, ddlArguments]))) createTable = Group(ddlWord("CREATE") + ddlWord("TABLE") + ddlTerm.setResultsName("tableName") + "(" + Group(delimitedList(ddlColumn)).setResultsName("columns") + ")").setResultsName("create") ddl = ZeroOrMore(SkipTo(createTable, True)) ddlComment = oneOf(["--", "#"]) + restOfLine ddl.ignore(ddlComment) # MAP SQL TYPES types = { 'tinyint': 'tinyint', 'smallint': 'smallint', 'integer': 'integer', 'int': 'integer', 'bigint': 'bigint', 'char': 'char_', 'varchar': 'varchar', 'text': 'text', 'tinyblob': 'blob', 'blob': 'blob', 'mediumblob': 'blob', 'longblob': 'blob',
def make_grammar_2(): """ Construct the BBDB grammar. See grammar.ebnf for the specification. """ # Helper functions for the brace types. LP, RP, LB, RB = map(Suppress, "()[]") Paren = lambda arg: LP + Group(arg) + RP Bracket = lambda arg: LB + Group(arg) + RB # Helper functions for constructing return types. def make_list(t): return t.asList() def make_dict(t): return {k: v for k, v in t[0] or []} def make_address_entry(t): return t[0].tag, { "location": list(t[0].location or []), "city": t[0].city or "", "state": t[0].state or "", "zipcode": t[0].zipcode or "", "country": t[0].country or "" } def make_record(t): return { "firstname": t[0].firstname, "lastname": t[0].lastname, "aka": t[0].aka or [], "company": t[0].company or "", "phone": t[0].phone or {}, "address": t[0].address or {}, "net": t[0].net or [], "fields": t[0].fields or {} } def make_string(t): return t[0][1:-1].replace(r'\"', '"') # Define the low-level entities. string = QuotedString(quoteChar='"', escChar='\\', unquoteResults=False) string.setParseAction(make_string) nil = Keyword("nil") nil.setParseAction(lambda t: [None]) atom = Word(alphanums + '-') dot = Suppress(Keyword(".")) integer = Word(nums) integer.setParseAction(lambda t: int(t[0])) # Phone. phone_usa = Group(OneOrMore(integer)) phone_nonusa = string phone_entry = Bracket(string("tag") + Or([phone_usa, phone_nonusa])) phone = Or([Paren(OneOrMore(phone_entry)), nil])("phone") phone.setParseAction(make_dict) # Address. location = Paren(OneOrMore(string))("location") location.setParseAction(make_list) address_entry = Bracket( string("tag") + location + string("city") + string("state") + string("zipcode") + string("country")) address_entry.setParseAction(make_address_entry) address = Or([Paren(OneOrMore(address_entry)), nil])("address") address.setParseAction(make_dict) # Field. field = Paren(atom + dot + string) fields = Or([Paren(OneOrMore(field)), nil])("fields") fields.setParseAction(make_dict) # Other parts of an entry. name = string("firstname") + Or([string("lastname"), nil]) company = Or([string, nil])("company") aka = Or([Paren(OneOrMore(string)), nil])("aka") aka.setParseAction(make_list) net = Or([Paren(OneOrMore(string)), nil])("net") net.setParseAction(make_list) cache = nil("cache") # A single record. record = Bracket(name + aka + company + phone + address + net + fields + cache) record.setParseAction(make_record) # All the records. bbdb = ZeroOrMore(record) bbdb.setParseAction(make_list) # Define comment syntax. comment = Regex(r";.*") bbdb.ignore(comment) return bbdb
def read_chain(s, struct_name, enum_def_parser, struct_def_parser, header_assignment_parser, struct_parsers): """Return a list of dictionaries with the contents of structures in a Yanny par file. :Parameters: - `s`: the string to parse - `struct_name`: the name of the struct to extract - `enum_def_parser`: a parser that parses one enum definition - `struct_def_parser`: a parser that parses on struct definition - `header_assignment_parser`: a parser that parses header assignments - `struct_parsers`: a dictionary of structures that parse structure data @return: a list of dictionaries with the contents of a chain >>> test_string = \"""# A sample file ... a foo ... bar baz ... ... typedef enum { ... OAK, ... MAPLE ... } TREETYPE; ... ... typedef struct { ... float x; ... int y ... } GOO ... ... typedef struct { ... int i; ... TREETYPE s ... } TREE; ... ... GOO 3.4 6 ... TREE 42 MAPLE ... TREE 44 OAK ... TREE 3 MAPLE ... GOO 4.22 103 ... \""" >>> >>> enum_def_parser = make_one_enum_def_parser() >>> type_parsers = parse_enum_defs(test_string, base_type_parsers) >>> struct_def_parser = make_one_struct_def_parser(type_parsers) >>> structs, struct_parsers = parse_struct_defs(test_string, type_parsers) >>> header_assignment_parser = make_one_header_assignment_parser(struct_parsers) >>> trees = read_chain(test_string, 'TREE', ... enum_def_parser, struct_def_parser, header_assignment_parser, struct_parsers) >>> print len(trees) 3 >>> print trees[1]['i'] 44 >>> print trees[2]['s'] MAPLE """ struct_name = struct_name.upper() other_list_parsers = [ struct_parsers[sn] for sn in struct_parsers.keys() if not sn == struct_name ] not_this_list_parser = Or( other_list_parsers + [enum_def_parser, header_assignment_parser, struct_def_parser] ).suppress() this_list_parser = ZeroOrMore(not_this_list_parser | struct_parsers[struct_name]) + stringEnd this_list_parser.ignore(hash_comment) this_list_parser.ignore(cStyleComment) raw_results = this_list_parser.parseString(s) results = [] for row_result in raw_results: dict_result = {} for field, value in row_result.items(): dict_result[field] = value results.append(dict_result) return results
class PulseProgram: """ Encapsulates a PulseProgrammer Program loadSource( filename ) loads the contents of the file The code is compiled in the following steps parse() generates self.code toBytecode() generates self.bytecode toBinary() generates self.binarycode the procedure updateVariables( dictionary ) updates variable values in the bytecode """ def __init__(self): self.variabledict = collections.OrderedDict() # keeps information on all variables to easily change them later self.labeldict = dict() # keep information on all labels self.source = collections.OrderedDict() # dictionary of source code files (stored as strings) self.code = [] # this is a list of lines self.bytecode = [] # list of op, argument tuples self.binarycode = bytearray() # binarycode to be uploaded self._exitcodes = dict() # generate a reverse dictionary of variables of type exitcode self.constDict = dict() class Board: channelLimit = 1 halfClockLimit = 500000000 self.adIndexList = [(x, 0) for x in range(6) ] self.adBoards = [ Board() ]*6 self.timestep = Q(20.0, 'ns') self.initBNF() def initBNF(self): constdecl = (CONST + NAME + VALUE).setParseAction(self.const_action) vardecl = (VAR + NAME + VALUE + Optional( COMMA + Regex("[^#\n]*")) ).setParseAction(self.var_action) insertdecl = (INSERT + dblQuotedString + LineEnd().suppress()).setParseAction(self.insert_action) LABEL = IDENTIFIER + COLON COMMANDEXP = (IDENTIFIER.setWhitespaceChars(" \t") + Regex("[^#\n]*").setWhitespaceChars(" \t") + LineEnd().suppress() ) COMMAND = COMMANDEXP.setParseAction(self.command_action) LABELEDCOMMAND = (LABEL + COMMANDEXP ).setParseAction(self.label_command_action) decl = constdecl | vardecl | insertdecl | LABELEDCOMMAND | COMMAND self.program = ZeroOrMore(decl) self.program.ignore(pythonStyleComment) def const_action( self, text, loc, arg ): """ add the const to the self.constDict dictionary """ logger = logging.getLogger(__name__) logger.debug("{0}:{1} const {2}".format(self.currentFile, lineno(loc, text), arg)) label, value = arg if label in self.constDict: logger.error( "Error parsing const in file '{0}': attempted to redefine'{1}' to '{2}' from '{3}'".format(self.currentFile, label, value, self.constDict[label]) ) raise ppexception("Redefining variable", self.currentFile, lineno, label) else: self.constDict[label] = int(value) def var_action( self, text, loc, arg): print("var_action", self.currentFile, lineno(loc, text), arg[0:2], arg[2].split(",") if len(arg)>2 else "") """ add a variable to the self.variablesdict """ logger = logging.getLogger(__name__) logger.debug( "{0}:{1} Variable {2}".format( self.currentFile, lineno(loc, text), arg ) ) var = Variable() label, data = arg[:2] fields = arg[2].split(",") if len(arg)>2 else [None]*3 fields += [None]*(3-len(fields)) var.type, unit, var.encoding = [ x if x is None or '' else x.strip() for x in fields ] var.name = label var.origin = self.currentFile var.enabled = True if var.encoding not in encodings: raise ppexception("unknown encoding {0} in file '{1}':{2}".format(var.encoding, self.currentFile, lineno(loc, text)), self.currentFile, lineno, var.encoding) try: data = str(eval(data, globals(), self.defines)) except Exception: logger.exception( "Evaluation error in file '{0}' on line: '{1}'".format(self.currentFile, data) ) if unit is not None: var.value = Q(float(data), unit) data = self.convertParameter( var.value, var.encoding ) else: var.value = Q(float(data)) data = int(round(float(data))) if label in self.defines: logger.error( "Error in file '%s': attempted to reassign '%s' to '%s' (from prev. value of '%s') in a var statement." %(self.currentFile, label, data, self.defines[label]) ) raise ppexception("variable redifinition", self.currentFile, lineno, label) else: self.defines[label] = label # add the variable to the dictionary of definitions to prevent identifiers and variables from having the same name # however, we do not want it replaced with a number but keep the name for the last stage of compilation pass var.data = data self.variabledict.update({ label: var}) if var.type == "exitcode": self._exitcodes[data & 0x0000ffff] = var def command_action( self, text, loc, arg): print("command_action", self.currentFile, lineno(loc, text), arg[0:1], arg[1].split(",") if len(arg)>1 else "") def label_command_action( self, text, loc, arg): print("label_command_action", self.currentFile, lineno(loc, text), arg[0:2], arg[2].split(",") if len(arg)>2 else "") def addLabel(self, label, address, sourcename, lineno): if label is not None: self.labeldict[label] = address def insert_action( self, text, loc, arg ): oldfile = self.currentFile print("insert_action", lineno(loc, text), arg) myprogram = self.program.copy() self.currentFile = arg[0][1:-1] result = myprogram.parseFile( self.currentFile ) self.currentFile = oldfile print(result) return result def assembleFile(self, filename): self.currentFile = filename result = self.program.parseFile( self.currentFile ) return result def setHardware(self, adIndexList, adBoards, timestep ): self.adIndexList = adIndexList self.adBoards = adBoards self.timestep = timestep assert self.timestep.has_dimension('s') def saveSource(self): for name, text in self.source.items(): with open(os.path.join(self.pp_dir, name), 'w') as f: f.write(text) def loadSource(self, pp_file): """ Load the source pp_file #include files are loaded recursively all code lines are added to self.sourcelines for each source file the contents are added to the dictionary self.source """ self.source.clear() self.pp_dir, self.pp_filename = os.path.split(pp_file) self.sourcelines = [] self.insertSource(self.pp_filename) self.compileCode() def updateVariables(self, variables ): """ update the variable values in the bytecode """ logger = logging.getLogger(__name__) for name, value in variables.items(): if name in self.variabledict: var = self.variabledict[name] address = var.address var.value = value logger.debug( "updateVariables {0} at address 0x{2:x} value {1}, 0x{3:x}".format(name, value, address, int(var.data)) ) var.data = self.convertParameter(value, var.encoding ) self.bytecode[address] = (self.bytecode[address][0], var.data ) self.variabledict[name] = var else: logger.error( "variable {0} not found in dictionary.".format(name) ) return self.bytecode def variables(self): mydict = dict() for name, var in self.variabledict.items(): mydict.update( {name: var.value }) return mydict def variable(self, variablename ): return self.variabledict.get(variablename).value def variableUpdateCode(self, variablename, value ): """returns the code to update the variable directly on the fpga consists of variablelocation and variablevalue """ var = self.variabledict[variablename] data = self.convertParameter(value, var.encoding ) return bytearray( struct.pack('II', (var.address, data))) def flattenList(self, l): return [item for sublist in l for item in sublist] def variableScanCode(self, variablename, values): var = self.variabledict[variablename] # [item for sublist in l for item in sublist] idiom for flattening of list return self.flattenList( [ (var.address, self.convertParameter(x, var.encoding)) for x in values ] ) def loadFromMemory(self): """Similar to loadSource only this routine loads from self.source """ self.sourcelines = [] self._exitcodes = dict() self.insertSource(self.pp_filename) self.compileCode() def toBinary(self): """ convert bytecode to binary """ logger = logging.getLogger(__name__) self.binarycode = bytearray() for wordno, (op, arg) in enumerate(self.bytecode): logger.debug( "{0} {1} {2} {3}".format( hex(wordno), hex(int(op)), hex(int(arg)), hex(int((int(op)<<24) + int(arg))) ) ) self.binarycode += struct.pack('I', int((op<<24) + arg)) return self.binarycode def currentVariablesText(self): lines = list() for name, var in iter(sorted(self.variabledict.items())): lines.append("{0} {1}".format(name, var.value)) return '\n'.join(lines) # routines below here should not be needed by the user insertPattern = re.compile('#insert\s+([\w.-_]+)', re.IGNORECASE) codelinePattern = re.compile('(#define|\s*[^#\s]+)', re.IGNORECASE) def insertSource(self, pp_file): """ read a source file pp_file calls itself recursively to for #insert adds the contents of this file to the dictionary self.source """ logger = logging.getLogger(__name__) if pp_file not in self.source: with open(os.path.join(self.pp_dir, pp_file)) as f: self.source[pp_file] = ''.join(f.readlines()) sourcecode = self.source[pp_file] for line, text in enumerate(sourcecode.splitlines()): m = self.insertPattern.match(text) if m: filename = m.group(1) logger.info( "inserting code from {0}".format(filename) ) self.insertSource(filename) else: if self.codelinePattern.match(text): self.sourcelines.append((text, line+1, pp_file)) labelPattern = re.compile('(\w+):\s+([^#\n\r]*)') opPattern = re.compile('\s*(\w+)(?:\s+([^#\n\r]*)){0,1}', re.IGNORECASE) varPattern = re.compile('var\s+(\w+)\s+([^#,\n\r]+)(?:,([^#,\n\r]+)){0,1}(?:,([^#,\n\r]+)){0,1}(?:,([^#,\n\r]+)){0,1}(?:#([^\n\r]+)){0,1}') # def parse(self): """ parse the code """ logger = logging.getLogger(__name__) self.code = [] self.variabledict = collections.OrderedDict() self.defines = dict() addr_offset = 0 for text, lineno, sourcename in self.sourcelines: m = self.varPattern.match(text) if m: self.addVariable(m, lineno, sourcename) else: m = self.definePattern.match(text) if m: self.addDefine(m, lineno, sourcename) else: # extract any JMP label, if present m = self.labelPattern.match(text) if m: label, text = m.groups() #so the operation after ":" still gets parsed CWC 08162012 else: label = None #The label for non-jump label line is NONE CWC 08172012 # search OPS list for a match to the current line m = self.opPattern.match(text) if m: op, args = m.groups() op = op.upper() # split and remove whitespace arglist = [0] if args is None else [ 0 if x is None else x.strip() for x in args.split(',')] #substitute the defined variable directly with the corresponding value CWC 08172012 arglist = [ self.defines[x] if x in self.defines else x for x in arglist ] #check for dds commands so CHAN commands can be inserted if (op[:3] == 'DDS'): try: board = self.adIndexList[int(arglist[0])][0] except ValueError: raise ppexception("DDS argument does not resolve to integer", sourcename, lineno, arglist[0]) chan = self.adIndexList[int(arglist[0])][1] if (self.adBoards[board].channelLimit != 1): #boards with more than one channel require an extra channel selection command chanData = self.adBoards[board].addCMD(chan) chanData += (int(board) << 16) self.code.append((len(self.code)+addr_offset, 'DDSCHN', chanData, label, sourcename, lineno)) data = arglist if len(arglist)>1 else arglist[0] self.addLabel( label, len(self.code), sourcename, lineno) self.code.append((len(self.code)+addr_offset, op, data, label, sourcename, lineno)) else: logger.error( "Error processing line {2}: '{0}' in file '{1}' (unknown opcode?)".format(text, sourcename, lineno) ) raise ppexception("Error processing line {2}: '{0}' in file '{1}' (unknown opcode?)".format(text, sourcename, lineno), sourcename, lineno, text) self.appendVariableCode() return self.code def appendVariableCode(self): """ append all variables to the instruction part of the code """ for var in list(self.variabledict.values()): address = len(self.code) self.code.append((address, 'NOP', var.data if var.enabled else 0, None, var.origin, 0 )) var.address = address def addVariable(self, m, lineno, sourcename): """ add a variable to the self.variablesdict """ logger = logging.getLogger(__name__) logger.debug( "Variable {0} {1} {2}".format( m.groups(), lineno, sourcename ) ) var = Variable() label, data, var.type, unit, var.encoding, var.comment = [ x if x is None else x.strip() for x in m.groups()] var.name = label var.origin = sourcename var.enabled = True if var.encoding not in encodings: raise ppexception("unknown encoding {0} in file '{1}':{2}".format(var.encoding, sourcename, lineno), sourcename, lineno, var.encoding) try: data = str(eval(data, globals(), self.defines)) except Exception: logger.exception( "Evaluation error in file '{0}' on line: '{1}'".format(sourcename, data) ) if unit is not None: var.value = Q(float(data), unit) data = self.convertParameter( var.value, var.encoding ) else: var.value = Q(float(data)) # var.value.output_prec(0) # without dimension the parameter has to be int. Thus, we do not want decimal places :) data = int(round(float(data))) if label in self.defines: logger.error( "Error in file '%s': attempted to reassign '%s' to '%s' (from prev. value of '%s') in a var statement." %(sourcename, label, data, self.defines[label]) ) raise ppexception("variable redifinition", sourcename, lineno, label) else: self.defines[label] = label # add the variable to the dictionary of definitions to prevent identifiers and variables from having the same name # however, we do not want it replaced with a number but keep the name for the last stage of compilation pass var.data = data self.variabledict.update({ label: var}) if var.type == "exitcode": self._exitcodes[data & 0x0000ffff] = var # code is (address, operation, data, label or variablename, currentfile) def toBytecode(self): """ generate bytecode from code """ logger = logging.getLogger(__name__) logger.debug( "\nCode ---> ByteCode:" ) self.bytecode = [] for line in self.code: logger.debug( "{0}: {1}".format(hex(line[0]), line[1:] )) bytedata = 0 if line[1] not in OPS: raise ppexception("Unknown command {0}".format(line[1]), line[4], line[5], line[1]) byteop = OPS[line[1]] try: data = line[2] #attempt to locate commands with constant data if (data == ''): #found empty data bytedata = 0 elif isinstance(data, (int, int)): bytedata = data elif isinstance(data, float): bytedata = int(data) elif isinstance(data, str): # now we are dealing with a variable and need its address bytedata = self.variabledict[line[2]].address if line[2] in self.variabledict else self.labeldict[line[2]] elif isinstance(data, list): # list is what we have for DDS, will have 8bit channel and 16bit address channel, data = line[2] if isinstance(data, str): data = self.variabledict[data].address bytedata = ((int(channel) & 0xf) << 16) | (int(data) & 0x0fff) except KeyError: logger.error( "Error assembling bytecode from file '{0}': Unknown variable: '{1}'. \n".format(line[4], data) ) raise ppexception("{0}: Unknown variable {1}".format(line[4], data), line[4], line[5], data) self.bytecode.append((byteop, bytedata)) logger.debug( "---> {0} {1}".format(hex(byteop), hex(bytedata)) ) return self.bytecode def convertParameter(self, mag, encoding=None ): """ convert a dimensioned parameter to the binary value expected by the hardware. The conversion is determined by the variable encoding """ if is_Q(mag): if mag.dimensionality == Dimensions.time: result = int(round(mag / self.timestep)) else: step, unit, _, mask = encodings[encoding] result = int(round(mag.m_as(unit) / step)) & mask else: if encoding: step, unit, _, mask = encodings[encoding] result = int(round(mag/step)) & mask else: result = mag return result def compileCode(self): self.parse() self.toBytecode() def exitcode(self, code): if code in self._exitcodes: var = self._exitcodes[code] if var.comment: return var.comment else: return var.name else: return "Exitcode {0} Not found".format(code)
include = INCLUDE + quotedString # -- the record itself record = brace_group(RECORDTYPE, field | include | c_code, types=(RTYP, )) # -- all top-level .dbd entries device_line = paren_group(DEVICE, RTYP, link_type, dset_name, quotedString) driver_line = paren_group(DRIVER, c_identifier) registrar_line = paren_group(REGISTRAR, c_identifier) function_line = paren_group(FUNCTION, c_identifier) variable_line = paren_group(VARIABLE, c_identifier, Optional(c_identifier, default='int')) top_level = Group(record | menu | device_line | driver_line | registrar_line | variable_line | function_line) dbd = ZeroOrMore(top_level) dbd.ignore(pythonStyleComment) # -- include files (supported only for including additional fields) dbd_include = ZeroOrMore(field | include | c_code) dbd_include.ignore(pythonStyleComment) def iter_results(info): check = [info] while check: c = check.pop(0) if c: if isinstance(c[0], str): yield c else: for other in c:
def from_gml(text): """ Return the list of graphs read from `text`, a string in GML format. This function assumes that the input follows the GML specification, provides unique integer ids even for isolated nodes, and defines one or more graphs. This function ignores anything other than node ids and edge endpoints. This means directed graphs are read as undirected graphs, node labels and edge weights are discarded, etc. If an edge endpoint (integer) is an unknown node id, the node is created. """ # Define the grammar with [pyparsing](http://pyparsing.wikispaces.com). # Don't use `from pyparsing import *` as it adds many constants # to the generated documentation. from pyparsing import ( srange, oneOf, Forward, Optional, Suppress, Word, ZeroOrMore, dblQuotedString, pythonStyleComment ) digit = srange("[0-9]") sign = Optional(oneOf("+ -")) mantissa = Optional("E" + sign + digit) # `Word(x)` is a sequence of one or more characters from the set x. digits = Word(digit) integer = sign + digits real = sign + Optional(digits) + "." + Optional(digits) + mantissa # For simplicity, use pyparsing's string with double-quotes, # hoping that it is a generalisation of GML's definition of a string. string = dblQuotedString # A GML file is a list of key-value pairs, where a value may be a list. # To handle this recursive definition, we delay what a pair is. pair = Forward() list = ZeroOrMore(pair) # A file may have comments, which are as in Python. Ignore them. list.ignore(pythonStyleComment) # `Word(x, y)` is 1 character from x followed by 0 or more from y. key = Word(srange("[a-zA-Z]"), srange("[a-zA-Z0-9]")) # `Suppress(x)` matches x but doesn't put it in the list of parsed tokens. listValue = Suppress("[") + list + Suppress("]") value = real | integer | string | listValue # The mandatory key-value pairs for graphs are as follows. graph = Suppress("graph") + listValue node = Suppress("node") + listValue anEdge = "edge" + listValue # to avoid conflict with edge() function id = Suppress("id") + integer source = Suppress("source") + integer target = Suppress("target") + integer # First try to parse graph-specific key-value pairs; otherwise ignore pair. pair <<= graph | node | anEdge | id | source | target | Suppress(key+value) # The above suppressions lead to the GML string # `'graph [ node [id 1 label "ego"] edge [source 1 target 1 weight 0.5] ]'` # being parsed into the list of tokens # `["1", "edge", "1", "1"]`, # which is converted by the following functions into a graph. def to_int(text, position, tokens): # Convert parsed integer tokens to integers, e.g. `["1"]` to `1`. return int(tokens[0]) def to_edge(text, position, tokens): # Assuming the above conversion was done, # convert `["edge", a, b]` to an edge incident to a and b. return edge(tokens[1], tokens[2]) def to_graph(text, position, tokens): # `tokens` is now a list of integers and edges, in any order. nodes = set() edges = set() for token in tokens: # If the token is an integer, it's a node id. if isinstance(token, int): nodes.add(token) else: edges.add(token) return network(edges, nodes) # Do the conversions as soon as the respective tokens are parsed. integer.setParseAction(to_int) anEdge.setParseAction(to_edge) graph.setParseAction(to_graph) # Parse the text with the main grammar rule. # Return the result as a list, not as a pyparsing object. return list.parseString(text).asList()
Group(OneOrMore(param_record)).setResultsName('records') + END param_stmt.setParseAction(ParamStmt) param_tabbing_stmt = Keyword("param") + param_default + ':' + Optional(symbol + ': ') + \ OneOrMore(data).setResultsName('params') \ + ':=' + ZeroOrMore(single).setResultsName('data') + END param_tabbing_stmt.setParseAction(ParamTabbingStmt) param_def_stmt = Keyword("param") + symbol + Optional(subscript_domain) + \ param_default + END param_def_stmt.setParseAction(ParamDefStmt) stmts = set_stmt | set_def_stmt | param_stmt | param_def_stmt | \ param_tabbing_stmt grammar = ZeroOrMore(stmts) + StringEnd() grammar.ignore("#" + SkipTo(lineEnd)) grammar.ignore("end;" + SkipTo(lineEnd)) class Amply(object): """ Data parsing interface """ def __init__(self, s=""): """ Create an Amply parser instance @param s (default ""): initial string to parse """ self.symbols = {}
release_site_definition = Group(identifier.setResultsName('name') + release_site_ + lbrace + dictOf(key,value).setResultsName('entries') + rbrace) object_definition = Group(identifier.setResultsName('compartmentName') + Suppress(object_) + (bracketidentifier | identifier) + (nestedExpr('{', '}',content=statement)).setResultsName('compartmentOptions')) hashed_initialization_section = Group(hashsymbol + Suppress(instantiate_) + identifier.setResultsName('name') + identifier.setResultsName('type') + lbrace + Group(ZeroOrMore(release_site_definition | object_definition)).setResultsName('entries') + rbrace ) other_sections = section_enclosure_ #statement = Group(identifier + equal + (quotedString | OneOrMore(mathElements))) + Suppress(LineEnd() | StringEnd()) grammar = ZeroOrMore(Suppress(other_sections) | Suppress(statement) | hashed_system_constants.setResultsName('systemConstants') | hashed_molecule_section.setResultsName('molecules') | hashed_reaction_section.setResultsName('reactions') | hashed_observable_section.setResultsName('observables') | hashed_initialization_section.setResultsName('initialization') | hashed_function_section.setResultsName('math_functions') #| Suppress(hashed_section) ) nonhashedgrammar = ZeroOrMore(Suppress(statement) | Suppress(hashed_section) | Dict(other_sections)) statementGrammar = ZeroOrMore(statement | Suppress(other_sections) | Suppress(hashed_section)) singleLineComment = "//" + restOfLine grammar.ignore(singleLineComment) grammar.ignore(cppStyleComment) nonhashedgrammar.ignore(singleLineComment) nonhashedgrammar.ignore(cppStyleComment) statementGrammar.ignore(singleLineComment) statementGrammar.ignore(cppStyleComment)
def get_parser(self): declaration = Forward() keyword = ( Keyword("enum") | Keyword("case") | Keyword("struct") | Keyword("default") | Keyword("switch") | Keyword("union") | Keyword("const") | Keyword("unsigned") | Keyword("int") | Keyword("hyper") | Keyword("float") | Keyword("double") | Keyword("bool") | Keyword("typedef") | Keyword("opaque") | Keyword("string") | Keyword("void") | Keyword("program") | Keyword("version") ) identifier = NotAny(keyword) + Word(alphas + alphas.upper(), alphanums + alphanums.upper() + "_", asKeyword=True) constant = Combine(Optional("-") + Word(nums)) constant.setParseAction(lambda s,l,t: [int(t[0])]) value = constant | identifier enum_body = Literal("{").suppress() + identifier + Literal("=").suppress() + value + ZeroOrMore(Literal(",").suppress() + identifier + Literal("=").suppress() + value) + Literal("}").suppress() enum_type_spec = Literal("enum").suppress() + enum_body enum_body.setParseAction(self.parse_enum) struct_body = Literal("{").suppress() + OneOrMore(declaration + Literal(";").suppress()) + Literal("}").suppress() struct_type_spec = Literal("struct").suppress() + struct_body struct_body.setParseAction(self.parse_struct) case_stmt = Literal("case").suppress() + value + Literal(":").suppress() + declaration + Literal(";").suppress() default_stmt = Literal("default") + Literal(":").suppress() + declaration + Literal(";").suppress() union_body = Literal("switch").suppress() + Literal("(").suppress() + declaration + Literal(")").suppress() + Literal("{").suppress() + Group(OneOrMore(Group(case_stmt)) + Optional(Group(default_stmt))) + Literal("}").suppress() union_type_spec = Literal("union").suppress() + union_body union_body.setParseAction(self.parse_union) constant_def = Literal("const").suppress() + identifier + Literal("=").suppress() + constant + Literal(";").suppress() constant_def.setParseAction(self.parse_const) type_spec = ( (Optional(Literal("unsigned")) + Literal("int")).setParseAction(self.parse_builtin) | (Optional(Literal("unsigned")) + Literal("hyper")).setParseAction(self.parse_builtin) | Literal("float").setParseAction(self.parse_builtin) | Literal("double").setParseAction(self.parse_builtin) | Literal("bool").setParseAction(self.parse_builtin) | enum_type_spec | struct_type_spec | union_type_spec | identifier ) proc_return = Literal("void") | type_spec procedure_def = proc_return + identifier + Literal("(").suppress() + (Literal("void") | type_spec) + ZeroOrMore(Literal(",").suppress() + type_spec) + Literal(")").suppress() + Literal("=").suppress() + constant + Literal(";").suppress() procedure_def.setParseAction(self.parse_procedure_def) version_def = Literal("version").suppress() + identifier + Literal("{").suppress() + OneOrMore(procedure_def) + Literal("}").suppress() + Literal("=").suppress() + constant + Literal(";").suppress() version_def.setParseAction(self.parse_version_def) program_body = Literal("{").suppress() + Group(OneOrMore(version_def)) + Literal("}").suppress() type_def = ( (Literal("typedef") + declaration + Literal(";")) | (Literal("enum") + identifier + enum_body + Literal(";")) | (Literal("struct") + identifier + struct_body + Literal(";")) | (Literal("union") + identifier + union_body + Literal(";")) | (Literal("program") + identifier + program_body + Literal("=").suppress() + constant + Literal(";")) ) type_def.setParseAction(self.parse_type_def) declaration << ( (type_spec + identifier + Literal("[") + value + Literal("]")) | (type_spec + identifier + Literal("<") + value + Literal(">")) | (type_spec + identifier) | (Literal("opaque") + identifier + Literal("[") + value + Literal("]")) | (Literal("opaque") + identifier + Literal("<") + value + Literal(">")) | (Literal("string") + identifier + Literal("<") + value + Literal(">")) | (type_spec + Literal("*") + identifier) | Literal("void") ) declaration.setParseAction(self.parse_decl) definition = type_def | constant_def specification = ZeroOrMore(definition) comment = (Literal("#") + restOfLine).suppress() specification.ignore(comment) return specification
from pyparsing import alphas, alphanums, hexnums, cppStyleComment, delimitedList, dblQuotedString, \ Suppress, Keyword, Word, Regex, Group, Optional, ZeroOrMore semi = Suppress(';') token_lbra = Suppress('{') token_rbra = Suppress('}') token_type = Keyword('byte') | Keyword('word') token_id = Word(alphas + '_', alphanums + '_') token_integer = Regex(r'[+-]?\d+') token_hex = Suppress('0x') + Word(hexnums) const_array = Group(Suppress('[') + delimitedList(token_hex) + Suppress(']')) const_value = token_integer | const_array var = Group(token_type + delimitedList(Group(token_id + Optional('=' + const_value)))) + semi call_func = Group(token_id + Suppress('()'))('call') + semi set_register = Group(token_hex + '=' + const_value) + semi body = ZeroOrMore(call_func | set_register | Group(dblQuotedString)) func = Group(Suppress('function') + token_id + Suppress('()') + token_lbra + Group(body) + token_rbra)('func') const = Group(Suppress('const') + token_type + Group(delimitedList( Group(token_id + '=' + const_value) )) + semi)('const') nes = ZeroOrMore(const | var | func) nes.ignore(cppStyleComment)
key = identifier + Suppress('=') value = restOfLine release_site_definition = Group(identifier.setResultsName('name') + release_site_ + lbrace + dictOf(key, value).setResultsName('entries') + rbrace) object_definition = Group(identifier.setResultsName('compartmentName') + Suppress(object_) + (bracketidentifier | identifier) + (nestedExpr('{', '}', content=statement)).setResultsName('compartmentOptions')) hashed_initialization_section = Group(hashsymbol + Suppress(instantiate_) + identifier.setResultsName('name') + identifier.setResultsName('type') + lbrace + Group(ZeroOrMore(release_site_definition | object_definition)).setResultsName('entries') + rbrace) other_sections = section_enclosure_ # statement = Group(identifier + equal + (quotedString | OneOrMore(mathElements))) + Suppress(LineEnd() | StringEnd()) grammar = ZeroOrMore(Suppress(other_sections) | Suppress(statement) | hashed_system_constants.setResultsName('systemConstants') | hashed_molecule_section.setResultsName('molecules') | hashed_reaction_section.setResultsName('reactions') | hashed_observable_section.setResultsName('observables') | hashed_initialization_section.setResultsName('initialization') | hashed_function_section.setResultsName('math_functions') # | Suppress(hashed_section) ) nonhashedgrammar = ZeroOrMore(Suppress(statement) | Suppress(hashed_section) | Dict(other_sections)) statementGrammar = ZeroOrMore(statement | Suppress(other_sections) | Suppress(hashed_section)) singleLineComment = "//" + restOfLine grammar.ignore(singleLineComment) grammar.ignore(cppStyleComment) nonhashedgrammar.ignore(singleLineComment) nonhashedgrammar.ignore(cppStyleComment) statementGrammar.ignore(singleLineComment) statementGrammar.ignore(cppStyleComment)
selectStatement << ( selectToken + columnNameList.setResultsName("columns") + fromToken + identifier.setResultsName("tablename") + Optional(whereClause) + Optional(orderByClause) + Optional(limitClause) ) BQLStatement = (selectStatement | createBtableStatement) + Optional(';') BQL = ZeroOrMore(BQLStatement) ## allows comments dashComment = "--" + restOfLine BQL.ignore(dashComment) def test( str ): print str,"->" try: tokens = BQL.parseString( str ) print "tokens = ", tokens print "tokens.tablename =", tokens.tablename print "tokens.filename =", tokens.filename #print "tokens.where =", tokens.where except ParseException, err: print " "*err.loc + "^\n" + err.msg print err print
def __init__(self, processor, baseiri, strict=False): """ See class docstring. """ # pylint: disable=R0914,R0915 self.reset(processor, baseiri, strict) PrefixedName = PNAME_LN | PNAME_NS Iri = IRIREF | PrefixedName BNode = BLANK_NODE_LABEL | ANON RDFLiteral = STRING + Optional(LANGTAG("langtag") | Group(Suppress("^^") + Iri)("datatype")) Object = Forward() Collection = Suppress("(") + ZeroOrMore(Object) + Suppress(")") PredicateObjectList = Forward() BlankNodePropertyList = Suppress("[") + PredicateObjectList + Suppress("]") TtlLiteral = RDFLiteral | NUMERIC_LITERAL | BOOLEAN_LITERAL Subject = Iri | BNode | Collection | VARIABLE # added for LD Patch Predicate = Iri Object << ( # pylint: disable=W0104 Iri | BNode | Collection | BlankNodePropertyList | TtlLiteral | VARIABLE ) # added for LD Patch Verb = Predicate | Keyword("a") ObjectList = Group(Object + ZeroOrMore(COMMA + Object)) PredicateObjectList << ( # pylint: disable=W0106 Verb + ObjectList + ZeroOrMore(SEMICOLON + Optional(Verb + ObjectList)) ) Triples = (Subject + PredicateObjectList) | (BlankNodePropertyList + Optional(PredicateObjectList)) Value = Iri | TtlLiteral | VARIABLE InvPredicate = Suppress("^") + Predicate Step = Suppress("/") + (Predicate | InvPredicate | INDEX) Filter = Forward() Constraint = Filter | UNICITY_CONSTRAINT Path = Group(OneOrMore(Step | Constraint)) Filter << ( Suppress("[") # pylint: disable=W0106 + Group(ZeroOrMore(Step | Constraint))("path") # = Path (*) + Optional(Suppress("=") + Object)("value") + Suppress("]") ) # (*) we can not reuse the Path rule defined above, # because we want to set a name for that component Turtle = Triples + ZeroOrMore(PERIOD + Triples) + Optional(PERIOD) Graph = Suppress("{") + Optional(Turtle) + Suppress("}") Prefix = Literal("@prefix") + PNAME_NS + IRIREF + PERIOD if not strict: SparqlPrefix = CaselessKeyword("prefix") + PNAME_NS + IRIREF Prefix = Prefix | SparqlPrefix Bind = BIND_CMD + VARIABLE + Value + Optional(Path) + PERIOD Add = ADD_CMD + Graph + PERIOD AddNew = ADDNEW_CMD + Graph + PERIOD Delete = DELETE_CMD + Graph + PERIOD DeleteExisting = DELETEEXISTING_CMD + Graph + PERIOD Cut = CUT_CMD + VARIABLE + PERIOD UpdateList = UPDATELIST_CMD + Subject + Predicate + SLICE + Collection + PERIOD Statement = Prefix | Bind | Add | AddNew | Delete | DeleteExisting | Cut | UpdateList Patch = ZeroOrMore(Statement) if not strict: Patch.ignore("#" + restOfLine) # Comment Patch.parseWithTabs() self.grammar = Patch IRIREF.setParseAction(self._parse_iri) PrefixedName.setParseAction(self._parse_pname) RDFLiteral.setParseAction(self._parse_turtleliteral) Collection.setParseAction(self._parse_collection) BlankNodePropertyList.setParseAction(self._parse_bnpl) Verb.setParseAction(self._parse_verb) ObjectList.setParseAction(self._parse_as_list) Triples.setParseAction(self._parse_tss) InvPredicate.setParseAction(self._parse_invpredicate) Filter.setParseAction(self._parse_filter) Path.setParseAction(self._parse_as_list) Prefix.setParseAction(self._do_prefix) Bind.setParseAction(self._do_bind) Add.setParseAction(self._do_add) AddNew.setParseAction(self._do_add_new) Delete.setParseAction(self._do_delete) DeleteExisting.setParseAction(self._do_delete_existing) Cut.setParseAction(self._do_cut) UpdateList.setParseAction(self._do_updatelist)
def parse_struct_defs(s, type_parsers=None): """Parse all the structure definitions in a string :Parameters: - `s`: the string to parse - `type_parsers`: a list of type parsers that can be used @return: a tuple with two dictionaries, one that describes the structs, the other stores the parsers >>> test_string = \"""# A sample file ... a foo ... bar baz ... ... typedef struct { # a comment ... float x; /* another comment */ ... int y ... } GOO ... ... typedef enum { ... START, ... END ... } RUNMARK; ... ... typedef struct { ... int i; ... RUNMARK m ... } MOO; ... ... GOO 3.4 6 ... \""" >>> >>> type_parsers = parse_enum_defs(test_string, base_type_parsers) >>> structs, struct_parsers = parse_struct_defs(test_string, type_parsers) >>> >>> print structs.keys() ['GOO', 'MOO'] >>> >>> print struct_parsers.keys() ['GOO', 'MOO'] >>> >>> for f in structs['GOO']: ... print "Field name: %-15s type name: %s" % (f['field_name'], f['type_name']) Field name: x type name: float Field name: y type name: int >>> >>> for f in structs['MOO']: ... print "Field name: %-15s type name: %s" % (f['field_name'], f['type_name']) Field name: i type name: int Field name: m type name: RUNMARK >>> >>> test_goo = "GOO 3.14 42" >>> test_parsed_goo = struct_parsers['GOO'].parseString(test_goo) >>> print "x: %f y: %d" % (test_parsed_goo[0]['x'], test_parsed_goo[0]['y']) x: 3.140000 y: 42 >>> >>> test_moo = "MOO 44 END" >>> test_parsed_moo = struct_parsers['MOO'].parseString(test_moo) >>> print "i: %d m: %s" % (test_parsed_moo[0]['i'], test_parsed_moo[0]['m']) i: 44 m: END """ if type_parsers is None: type_parsers = base_type_parsers one_struct_def_parser = make_one_struct_def_parser(type_parsers) not_struct = ((value_name | enum_declaration_start | possible_type_name | right_brace) + restOfLine).suppress() struct_def_parser = ZeroOrMore(Group(one_struct_def_parser) | not_struct) + stringEnd struct_def_parser.ignore(hash_comment) struct_def_parser.ignore(cStyleComment) struct = {} struct_parser = {} for this_struct in struct_def_parser.parseString(s): struct_name = this_struct['struct_name'].upper() struct[struct_name] = \ [{'field_name': f['field_name'], 'type_name': f['type_name']} for f in this_struct['fields']] struct_parser[struct_name] = \ Group(CaselessKeyword(this_struct['struct_name']) \ + And( [linecont+type_parsers[f['type_name']](f['field_name']) for f in this_struct['fields']] )) return struct, struct_parser
class SFZParser(object): def __init__(self, path, text, state=None): self.path = path self.base_path = os.path.dirname(path) self.text = text self.state = state opcode_name = Word(alphanums + '_') value = Regex(r'.*?(?=\s*(([a-zA-Z0-9_]+=)|//|<[a-z]|$))', re.MULTILINE) opcode = locatedExpr(opcode_name) + Literal('=').suppress() + value opcode.setParseAction(self.handle_opcode) section_name = Literal('<').suppress() + Word(alphas) + Literal( '>').suppress() section = section_name section.setParseAction(self.handle_section) include = Literal('#include').suppress() + locatedExpr( QuotedString('"')) include.setParseAction(self.handle_include) statement = (section ^ opcode ^ include) self.sfz_file = ZeroOrMore(statement) + stringEnd comment = Literal('//') + restOfLine self.sfz_file.ignore(comment) def handle_include(self, s, loc, toks): path = os.path.join(self.base_path, normalize_path(toks[0].value)) try: with open(path) as fp: f = fp.read() except IOError as exc: raise IncludeException(s, loc=toks[0].locn_start, msg=str(exc)) subparser = SFZParser(path, f, self.state) subparser.sfz_file.parseString(f) def handle_section(self, s, loc, toks): name = toks[0] if name == 'region': section = Region(self.state.instr, name, group=self.state.current_group, control=self.state.current_control) self.state.instr.regions.append(section) elif name == 'group': section = Section(self.state.instr, name) self.state.current_group = section elif name == 'control': section = Section(self.state.instr, name) self.state.current_control = section else: raise InvalidSectionException(s, loc, "Invalid section name '%s'" % name) self.state.current_section = section def handle_opcode(self, s, loc, toks): loc = toks[0].locn_start name = toks[0].value try: opdef = opmap[name] except KeyError: raise UnknownOpCodeException(s, loc=loc, msg="Unknown opcode '%s'" % key) try: value = opdef.parse_value(toks[1]) except ValueError as exc: raise InvalidValueException( s, loc=loc, msg="Invalid value for opcode '%s': %s" % (key, str(exc))) self.state.current_section._opcodes[name] = value self.state.current_section._opcode_locs[name] = (s, loc) def parse(self): self.state = ParserState() self.state.instr = Instrument(os.path.abspath(self.path)) self.sfz_file.parseString(self.text) for region in self.state.instr.regions: if not os.path.isfile(region.sample): s, loc = region.get_opcode_loc('sample') raise SampleMissingException( s, loc, "Missing sample '%s'" % region.sample) return self.state.instr
class TOMLParser(object): def __init__(self): key_name = Word(re.sub(r"[\[\]=\"]", "", printables)) kgrp_name = Word(re.sub(r"[\[\]\.]", "", printables)) basic_int = Optional("-") + ("0" | Word(nums)) types = dict( string = QuotedString("\"", escChar="\\"), integer = Combine(basic_int), float = Combine(basic_int + "." + Word(nums)), datetime = Regex(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z"), boolean = Keyword("true") | Keyword("false"), array = Forward(), ) pure_array = Or(delimitedList(type_) for type_ in types.values()) types["array"] << Group(Suppress("[") + Optional(pure_array) + Suppress("]")) value = Or(type_ for type_ in types.values()) keyvalue = key_name + Suppress("=") + value + Suppress(LineEnd()) keygroup_namespace = kgrp_name + ZeroOrMore(Suppress(".") + kgrp_name) keygroup = "[" + keygroup_namespace + "]" + LineEnd() comments = pythonStyleComment self._toplevel = ZeroOrMore(keyvalue | keygroup) self._toplevel.ignore(comments) for k, v in types.items(): v.setParseAction(getattr(self, "_parse_"+k)) keyvalue.setParseAction(self._parse_keyvalue) keygroup_namespace.setParseAction(self._parse_keygroup_namespace) def _parse_string(self, src, loc, toks): match = re.search(r"(?<!\\)(\\[^0tnr\"\\])", toks[0]) if match: raise ParseException("Reserved escape sequence \"%s\"" % match.group(), loc) return unescape(toks[0]) _parse_integer = lambda self, tok: int(tok[0]) _parse_float = lambda self, tok: float(tok[0]) _parse_boolean = lambda self, tok: bool(tok[0]) ISO8601 = "%Y-%m-%dT%H:%M:%SZ" def _parse_datetime(self, src, loc, toks): try: return datetime.strptime(toks[0], self.ISO8601) except ValueError: # this informative error message will never make it out because # pyparsing catches ParseBaseException and reraises on its own. # oh well. raise ParseException("invalid datetime \"%s\"" % toks[0], loc) _parse_array = lambda self, tok: [tok[0]] def _parse_keyvalue(self, s, loc, toks): k, v = toks.asList() if k in self._cur: raise ParseException("key %s already exists" % k, loc) self._cur[k] = v def _parse_keygroup_namespace(self, s, loc, toks): cur = self._root for subname in toks: subspace = cur.get(subname, {}) if not isinstance(subspace, dict): raise ParseException("key %s already exists" % subname, loc) cur = cur.setdefault(subname, subspace) self._cur = cur def parse(self, s): self._root = {} self._cur = self._root self._toplevel.parseWithTabs() self._toplevel.parseString(s, parseAll=True) return self._root
def SPICE_BNF(): global bnf if not bnf: # punctuation colon = Literal(":").suppress() lbrace = Literal("{").suppress() rbrace = Literal("}").suppress() lbrack = Literal("[").suppress() rbrack = Literal("]").suppress() lparen = Literal("(").suppress() rparen = Literal(")").suppress() equals = Literal("=").suppress() comma = Literal(",").suppress() semi = Literal(";").suppress() # primitive types int8_ = Keyword("int8").setParseAction(replaceWith(ptypes.int8)) uint8_ = Keyword("uint8").setParseAction(replaceWith(ptypes.uint8)) int16_ = Keyword("int16").setParseAction(replaceWith(ptypes.int16)) uint16_ = Keyword("uint16").setParseAction(replaceWith(ptypes.uint16)) int32_ = Keyword("int32").setParseAction(replaceWith(ptypes.int32)) uint32_ = Keyword("uint32").setParseAction(replaceWith(ptypes.uint32)) int64_ = Keyword("int64").setParseAction(replaceWith(ptypes.int64)) uint64_ = Keyword("uint64").setParseAction(replaceWith(ptypes.uint64)) # keywords enum32_ = Keyword("enum32").setParseAction(replaceWith(32)) enum16_ = Keyword("enum16").setParseAction(replaceWith(16)) enum8_ = Keyword("enum8").setParseAction(replaceWith(8)) flags32_ = Keyword("flags32").setParseAction(replaceWith(32)) flags16_ = Keyword("flags16").setParseAction(replaceWith(16)) flags8_ = Keyword("flags8").setParseAction(replaceWith(8)) channel_ = Keyword("channel") server_ = Keyword("server") client_ = Keyword("client") protocol_ = Keyword("protocol") typedef_ = Keyword("typedef") struct_ = Keyword("struct") message_ = Keyword("message") image_size_ = Keyword("image_size") bytes_ = Keyword("bytes") cstring_ = Keyword("cstring") switch_ = Keyword("switch") default_ = Keyword("default") case_ = Keyword("case") identifier = Word(alphas, alphanums + "_") enumname = Word(alphanums + "_") integer = ( (Combine(CaselessLiteral("0x") + Word(nums + "abcdefABCDEF")) | Word(nums + "+-", nums)) .setName("int") .setParseAction(cvtInt) ) typename = identifier.copy().setParseAction(lambda toks: ptypes.TypeRef(str(toks[0]))) # This is just normal "types", i.e. not channels or messages typeSpec = Forward() attributeValue = integer ^ identifier attribute = Group(Combine("@" + identifier) + Optional(lparen + delimitedList(attributeValue) + rparen)) attributes = Group(ZeroOrMore(attribute)) arraySizeSpecImage = Group(image_size_ + lparen + integer + comma + identifier + comma + identifier + rparen) arraySizeSpecBytes = Group(bytes_ + lparen + identifier + comma + identifier + rparen) arraySizeSpecCString = Group(cstring_ + lparen + rparen) arraySizeSpec = ( lbrack + Optional( identifier ^ integer ^ arraySizeSpecImage ^ arraySizeSpecBytes ^ arraySizeSpecCString, default="" ) + rbrack ) variableDef = Group( typeSpec + Optional("*", default=None) + identifier + Optional(arraySizeSpec, default=None) + attributes - semi ).setParseAction(parseVariableDef) switchCase = Group( Group( OneOrMore( default_.setParseAction(replaceWith(None)) + colon | Group(case_.suppress() + Optional("!", default="") + identifier) + colon ) ) + variableDef ).setParseAction(lambda toks: ptypes.SwitchCase(toks[0][0], toks[0][1])) switchBody = Group( switch_ + lparen + delimitedList(identifier, delim=".", combine=True) + rparen + lbrace + Group(OneOrMore(switchCase)) + rbrace + identifier + attributes - semi ).setParseAction(lambda toks: ptypes.Switch(toks[0][1], toks[0][2], toks[0][3], toks[0][4])) messageBody = structBody = Group(lbrace + ZeroOrMore(variableDef | switchBody) + rbrace) structSpec = Group(struct_ + identifier + structBody + attributes).setParseAction( lambda toks: ptypes.StructType(toks[0][1], toks[0][2], toks[0][3]) ) # have to use longest match for type, in case a user-defined type name starts with a keyword type, like "channel_type" typeSpec << ( structSpec ^ int8_ ^ uint8_ ^ int16_ ^ uint16_ ^ int32_ ^ uint32_ ^ int64_ ^ uint64_ ^ typename ).setName("type") flagsBody = enumBody = Group( lbrace + delimitedList(Group(enumname + Optional(equals + integer))) + Optional(comma) + rbrace ) messageSpec = ( Group(message_ + messageBody + attributes).setParseAction( lambda toks: ptypes.MessageType(None, toks[0][1], toks[0][2]) ) | typename ) channelParent = Optional(colon + typename, default=None) channelMessage = Group( messageSpec + identifier + Optional(equals + integer, default=None) + semi ).setParseAction(lambda toks: ptypes.ChannelMember(toks[0][1], toks[0][0], toks[0][2])) channelBody = channelParent + Group( lbrace + ZeroOrMore(server_ + colon | client_ + colon | channelMessage) + rbrace ) enum_ = enum32_ | enum16_ | enum8_ flags_ = flags32_ | flags16_ | flags8_ enumDef = Group(enum_ + identifier + enumBody + attributes - semi).setParseAction( lambda toks: ptypes.EnumType(toks[0][0], toks[0][1], toks[0][2], toks[0][3]) ) flagsDef = Group(flags_ + identifier + flagsBody + attributes - semi).setParseAction( lambda toks: ptypes.FlagsType(toks[0][0], toks[0][1], toks[0][2], toks[0][3]) ) messageDef = Group(message_ + identifier + messageBody + attributes - semi).setParseAction( lambda toks: ptypes.MessageType(toks[0][1], toks[0][2], toks[0][3]) ) channelDef = Group(channel_ + identifier + channelBody + attributes - semi).setParseAction( lambda toks: ptypes.ChannelType(toks[0][1], toks[0][2], toks[0][3], toks[0][4]) ) structDef = Group(struct_ + identifier + structBody + attributes - semi).setParseAction( lambda toks: ptypes.StructType(toks[0][1], toks[0][2], toks[0][3]) ) typedefDef = Group(typedef_ + identifier + typeSpec + attributes - semi).setParseAction( lambda toks: ptypes.TypeAlias(toks[0][1], toks[0][2], toks[0][3]) ) definitions = typedefDef | structDef | enumDef | flagsDef | messageDef | channelDef protocolChannel = Group(typename + identifier + Optional(equals + integer, default=None) + semi).setParseAction( lambda toks: ptypes.ProtocolMember(toks[0][1], toks[0][0], toks[0][2]) ) protocolDef = Group( protocol_ + identifier + Group(lbrace + ZeroOrMore(protocolChannel) + rbrace) + semi ).setParseAction(lambda toks: ptypes.ProtocolType(toks[0][1], toks[0][2])) bnf = ZeroOrMore(definitions) + protocolDef + StringEnd() singleLineComment = "//" + restOfLine bnf.ignore(singleLineComment) bnf.ignore(cStyleComment) return bnf
pluginName = Word(alphanums + "@_") pluginDesc = Group(pluginType + pluginName + lbrace + Group(ZeroOrMore(pluginAttr)) + rbrace).setParseAction( getPluginDesc ) pluginDesc.ignore("//" + restOfLine) pluginDesc.ignore(cStyleComment) # Scene # sceneDesc = OneOrMore(pluginDesc) sceneDesc.ignore("//" + restOfLine) sceneDesc.ignore(cStyleComment) nameParser = ZeroOrMore(Group(pluginType + pluginName + lbrace)) nameParser.ignore("//" + restOfLine) nameParser.ignore(cStyleComment) def ParseVrscene(filepath): return sceneDesc.parseString(open(filepath, "r").read()) def GetMaterialsNames(filepath): materialPluginNames = [] with open(filepath, "r") as f: for l in f: result = nameParser.parseString(l) if result: res = result[0] if res[0].startswith("Mtl"):
ddlUnsigned, ddlNotNull, ddlAutoValue, ddlDefaultValue, ddlFunctionWord("NOW"), ddlTerm, ddlNum, ddlColumnComment, ddlString, ddlArguments, ddlMathCombined, ddlInlineComment ]))) ddlIfNotExists = Optional(Group(ddlWord("IF") + ddlWord("NOT") + ddlWord( "EXISTS")).setResultsName("ifNotExists")) createTable = Group(ddlWord("CREATE") + ddlWord("TABLE") + ddlIfNotExists + ddlName.setResultsName("tableName") + "(" + Group( delimitedList(ddlColumn)).setResultsName( "columns") + ")").setResultsName("create") #ddlString.setDebug(True) #uncomment to debug pyparsing ddl = ZeroOrMore(Suppress(SkipTo(createTable, False)) + createTable) ddlComment = oneOf(["--", "#"]) + restOfLine ddl.ignore(ddlComment) # MAP SQL TYPES types = { 'tinyint': 'tinyint', 'smallint': 'smallint', 'smallserial': 'smallint', # PostgreSQL 'integer': 'integer', 'int': 'integer', 'serial': 'integer', # PostgreSQL 'mediumint': 'integer', 'bigint': 'bigint', 'bigserial': 'bigint', # PostgreSQL 'char': 'char_', 'varchar': 'varchar', 'text': 'text',
def parser_bnf(): """Grammar for parsing podcast configuration files.""" at = Literal("@").suppress() caret = Literal("^") colon = Literal(":").suppress() left_bracket = Literal("[").suppress() period = Literal(".").suppress() right_bracket = Literal("]").suppress() # zero_index ::= [0-9]+ zero_index = Word(nums).setParseAction(lambda s, l, t: int(t[0])) # filename ::= [A-Za-z0-9][-A-Za-z0-9._ ]+ filename_first = Word(alphanums, exact=1) filename_rest = Word(alphanums + "-_/. ") filename = Combine(filename_first + Optional(filename_rest)) # millisecs ::= "." [0-9]+ millisecs = (Word(nums).setParseAction( lambda s, l, t: int(t[0][:3].ljust(3, "0"))).setResultsName("ms")) # hours, minutes, seconds ::= zero_index hours = zero_index.setResultsName("hh") minutes = zero_index.setResultsName("mm") seconds = zero_index.setResultsName("ss") hours_minutes = hours + colon + minutes + colon | minutes + colon secs_millisecs = (seconds + Optional(period + millisecs) | period + millisecs) # timestamp ::= [[hours ":"] minutes ":"] seconds ["." millisecs] timestamp = Optional(hours_minutes) + secs_millisecs # duration_file ::= "@", filename # We need a separate item for a lonely duration file timestamp so # that we can attach a parse action just to the lonely case. Using # duration_file alone means the parse action is attached to all # instances of duration_file. duration_file = at + filename.setResultsName("filename") lonely_duration_file = at + filename.setResultsName("filename") # timespecs ::= timestamp [duration_file | {timestamp}] # If duration_file timestamp is lonely, prepend a zero timestamp. timespecs = Or([ lonely_duration_file.setParseAction( lambda s, l, t: [timestamp.parseString("00:00:00.000"), t]), Group(timestamp) + duration_file, OneOrMore(Group(timestamp.setParseAction(default_timestamp_fields))) ]) # last_frame ::= "-1" | "last" last_frame = oneOf(["-1", "last"]).setParseAction(replaceWith(-1)) # frame_number ::= ":" (zero_index | last_frame) frame_number = colon - (zero_index | last_frame).setResultsName("num") # stream_number ::= ":" zero_index stream_number = colon - zero_index.setResultsName("num") # input_file ::= ":" [filename] input_file = colon - Optional(filename).setResultsName("filename") # previous_segment ::= ":" "^" previous_segment = colon - caret.setResultsName("filename") # frame_input_file ::= input_file | previous_segment frame_input_file = Or([input_file, previous_segment]) # av_trailer ::= input_file [stream_number] av_trailer = input_file + Optional(stream_number) # frame_type ::= "frame" | "f" frame_type = oneOf(["f", "frame"]).setParseAction(replaceWith("frame")) # frame_input ::= frame_type [frame_input_file [frame_number]] frame_input = (frame_type.setResultsName("type") + Optional(frame_input_file + Optional(frame_number))) # video_type ::= "video" | "v" video_type = oneOf(["v", "video"]).setParseAction(replaceWith("video")) # audio_type ::= "audio" | "a" audio_type = oneOf(["a", "audio"]).setParseAction(replaceWith("audio")) # av_input ::= (audio_type | video_type) [av_trailer] av_input = ((audio_type | video_type).setResultsName("type") + Optional(av_trailer)) # inputspec ::= "[" (av_input | frame_input) "]" inputspec = (left_bracket + delimitedList( av_input | frame_input, delim=":").setParseAction(default_input_fields) - right_bracket) # segmentspec ::= inputspec [timespecs] segmentspec = Group(inputspec + Group(Optional(timespecs)).setResultsName("times")) # config ::= {segmentspec} config = ZeroOrMore(segmentspec) config.ignore(pythonStyleComment) return config