def indentedBlock(expr, indent_stack, indent=True): """Define space-delimited indentation blocks. Helper method for defining space-delimited indentation blocks, such as those used to define block statements in Python source code. There is also a version in pyparsing but doesn't seem to be working fine with JSONAlchemy cfg files. """ def check_sub_indent(string, location, tokens): """Check the indentation.""" cur_col = col(location, string) if cur_col > indent_stack[-1]: indent_stack.append(cur_col) else: raise ParseException(string, location, "not a subentry") def check_unindent(string, location, tokens): """Check the 'undentation'.""" if location >= len(string): return cur_col = col(location, string) if not (cur_col < indent_stack[-1] and cur_col <= indent_stack[-2]): raise ParseException(string, location, "not an unindent") def do_unindent(): """Unindent.""" indent_stack.pop() indent = lineEnd.suppress() + empty + empty.copy()\ .setParseAction(check_sub_indent) undent = FollowedBy(empty).setParseAction(check_unindent) undent.setParseAction(do_unindent) return indent + expr + undent
def indentedBlock(expr, indent_stack, indent=True): """Define space-delimited indentation blocks. Helper method for defining space-delimited indentation blocks, such as those used to define block statements in Python source code. There is also a version in pyparsing but doesn't seem to be working fine with JSONAlchemy cfg files. """ def check_sub_indent(string, location, tokens): """Check the indentation.""" cur_col = col(location, string) if cur_col > indent_stack[-1]: indent_stack.append(cur_col) else: raise ParseException(string, location, "not a subentry") def check_unindent(string, location, tokens): """Check the 'undentation'.""" if location >= len(string): return cur_col = col(location, string) if not(cur_col < indent_stack[-1] and cur_col <= indent_stack[-2]): raise ParseException(string, location, "not an unindent") def do_unindent(): """Unindent.""" indent_stack.pop() indent = lineEnd.suppress() + empty + empty.copy()\ .setParseAction(check_sub_indent) undent = FollowedBy(empty).setParseAction(check_unindent) undent.setParseAction(do_unindent) return indent + expr + undent
def parse_pabl(self, raw_pabl): INDENT = lineEnd.suppress() + empty + empty.copy().setParseAction( self.check_sub_indent) UNDENT = FollowedBy(empty).setParseAction(self.check_unindent) UNDENT.setParseAction(self.unindent) terminator = Literal(';').suppress() comment = Literal('#') + restOfLine item_name = Word(alphas, alphanums + '_') variable = Word(alphas, alphanums + '_.') variable_as = (variable + 'as' + item_name) stmt = Forward() suite = Group( OneOrMore(empty + stmt.setParseAction(self.check_peer_indent))) suite.ignore(comment) item_start = Literal('@item').suppress() item_end = Literal(':').suppress() permission_start = Literal('@permissions') item_decl = (item_start + item_name.setResultsName('item') + item_end) item_defn = Group(item_decl + INDENT + suite + UNDENT) permission_decl = (permission_start + Group( delimitedList(item_name).setResultsName('permissions')) + item_end) permission_defn = Group(permission_decl + INDENT + suite + UNDENT) fieldList = delimitedList( Group(variable_as) | variable ).setResultsName('fields') + terminator stmt << (item_defn | fieldList | Group(permission_defn)) parseTree = suite.parseString(raw_pabl) return parseTree
def _create_field_parser(): """ Creates a parser using pyparsing that works with bibfield rule definitions BNF like grammar: rule ::= ([persitent_identifier] json_id ["[0]" | "[n]"] "," aliases":" INDENT body UNDENT) | include | python_comment include ::= "include(" PATH ")" body ::= [inherit_from] (creator | derived | calculated) [checker] [documentation] [producer] aliases ::= json_id ["[0]" | "[n]"] ["," aliases] creator ::= "creator:" INDENT creator_body+ UNDENT creator_body ::= [decorators] source_format "," source_tag "," python_allowed_expr source_format ::= MASTER_FORMATS source_tag ::= QUOTED_STRING derived ::= "derived" INDENT derived_calculated_body UNDENT calculated ::= "calculated:" INDENT derived_calculated_body UNDENT derived_calculated_body ::= [decorators] "," python_allowed_exp decorators ::= (peristent_identfier | legacy | do_not_cache | parse_first | depends_on | only_if | only_if_master_value)* peristent_identfier ::= @persitent_identifier( level ) legacy ::= "@legacy(" correspondences+ ")" correspondences ::= "(" source_tag [ "," tag_name ] "," json_id ")" parse_first ::= "@parse_first(" jsonid+ ")" depends_on ::= "@depends_on(" json_id+ ")" only_if ::= "@only_if(" python_condition+ ")" only_if_master_value ::= "@only_if_master_value(" python_condition+ ")" inherit_from ::= "@inherit_from()" python_allowed_exp ::= ident | list_def | dict_def | list_access | dict_access | function_call checker ::= "checker:" INDENT checker_function+ UNDENT documentation ::= INDENT doc_string subfield* UNDENT doc_string ::= QUOTED_STRING subfield ::= "@subfield" json_id["."json_id*] ":" docstring producer ::= "producer:" INDENT producer_body UNDENT producer_body ::= producer_code "," python_dictionary producer_code ::= ident """ indent_stack = [1] def check_sub_indent(str, location, tokens): cur_col = col(location, str) if cur_col > indent_stack[-1]: indent_stack.append(cur_col) else: raise ParseException(str, location, "not a subentry") def check_unindent(str, location, tokens): if location >= len(str): return cur_col = col(location, str) if not(cur_col < indent_stack[-1] and cur_col <= indent_stack[-2]): raise ParseException(str, location, "not an unindent") def do_unindent(): indent_stack.pop() INDENT = lineEnd.suppress() + empty + empty.copy().setParseAction(check_sub_indent) UNDENT = FollowedBy(empty).setParseAction(check_unindent) UNDENT.setParseAction(do_unindent) json_id = (Word(alphas + "_", alphanums + "_") + Optional(oneOf("[0] [n]")))\ .setResultsName("json_id", listAllMatches=True)\ .setParseAction(lambda tokens: "".join(tokens)) aliases = delimitedList((Word(alphanums + "_") + Optional(oneOf("[0] [n]"))) .setParseAction(lambda tokens: "".join(tokens)))\ .setResultsName("aliases") ident = Word(alphas + "_", alphanums + "_") dict_def = originalTextFor(nestedExpr('{', '}')) list_def = originalTextFor(nestedExpr('[', ']')) dict_access = list_access = originalTextFor(ident + nestedExpr('[', ']')) function_call = originalTextFor(ZeroOrMore(ident + ".") + ident + nestedExpr('(', ')')) python_allowed_expr = (dict_def ^ list_def ^ dict_access ^ \ list_access ^ function_call ^ restOfLine)\ .setResultsName("value", listAllMatches=True) persistent_identifier = (Suppress("@persistent_identifier") + \ nestedExpr("(", ")"))\ .setResultsName("persistent_identifier") legacy = (Suppress("@legacy") + originalTextFor(nestedExpr("(", ")")))\ .setResultsName("legacy", listAllMatches=True) only_if = (Suppress("@only_if") + originalTextFor(nestedExpr("(", ")")))\ .setResultsName("only_if") only_if_master_value = (Suppress("@only_if_value") + \ originalTextFor(nestedExpr("(", ")")))\ .setResultsName("only_if_master_value") depends_on = (Suppress("@depends_on") + \ originalTextFor(nestedExpr("(", ")")))\ .setResultsName("depends_on") parse_first = (Suppress("@parse_first") + \ originalTextFor(nestedExpr("(", ")")))\ .setResultsName("parse_first") memoize = (Suppress("@memoize") + nestedExpr("(", ")"))\ .setResultsName("memoize") field_decorator = parse_first ^ depends_on ^ only_if ^ \ only_if_master_value ^ memoize ^ legacy #Independent decorators inherit_from = (Suppress("@inherit_from") + \ originalTextFor(nestedExpr("(", ")")))\ .setResultsName("inherit_from") override = (Suppress("@") + "override")\ .setResultsName("override") extend = (Suppress("@") + "extend")\ .setResultsName("extend") master_format = (Suppress("@master_format") + \ originalTextFor(nestedExpr("(", ")")))\ .setResultsName("master_format") \ .setParseAction(lambda toks: toks[0]) derived_calculated_body = (ZeroOrMore(field_decorator) + python_allowed_expr)\ .setResultsName('derived_calculated_def') derived = "derived" + Suppress(":") + \ INDENT + derived_calculated_body + UNDENT calculated = "calculated" + Suppress(":") + \ INDENT + derived_calculated_body + UNDENT source_tag = quotedString\ .setParseAction(removeQuotes)\ .setResultsName("source_tag", listAllMatches=True) source_format = Word(alphas, alphanums + "_")\ .setResultsName("source_format", listAllMatches=True) creator_body = (ZeroOrMore(field_decorator) + source_format + \ Suppress(",") + source_tag + Suppress(",") + python_allowed_expr)\ .setResultsName("creator_def", listAllMatches=True) creator = "creator" + Suppress(":") + \ INDENT + OneOrMore(creator_body) + UNDENT field_def = (creator | derived | calculated)\ .setResultsName("type_field", listAllMatches=True) #JsonExtra json_dumps = (Suppress('dumps') + Suppress(',') + python_allowed_expr)\ .setResultsName("dumps")\ .setParseAction(lambda toks: toks.value[0]) json_loads = (Suppress("loads") + Suppress(",") + python_allowed_expr)\ .setResultsName("loads")\ .setParseAction(lambda toks: toks.value[0]) json_extra = (Suppress('json:') + \ INDENT + Each((json_dumps, json_loads)) + UNDENT)\ .setResultsName('json_ext') #Checker checker_function = (Optional(master_format) + ZeroOrMore(ident + ".") + ident + originalTextFor(nestedExpr('(', ')')))\ .setResultsName("checker", listAllMatches=True) checker = ("checker" + Suppress(":") + INDENT + OneOrMore(checker_function) + UNDENT) #Description/Documentation doc_double = QuotedString(quoteChar='"""', multiline=True) doc_single = QuotedString(quoteChar="'''", multiline=True) doc_string = INDENT + (doc_double | doc_single) + UNDENT description_body = (Suppress('description:') + doc_string).\ setParseAction(lambda toks: toks[0][0]) description = (description_body | doc_double | doc_single)\ .setResultsName('description') #Producer producer_code = (Word(alphas, alphanums + "_")\ + originalTextFor(nestedExpr("(", ")")))\ .setResultsName('producer_code', listAllMatches=True) producer_body = (producer_code + Suppress(",") + python_allowed_expr)\ .setResultsName("producer_rule", listAllMatches=True) producer = Suppress("producer:") + INDENT + OneOrMore(producer_body) + UNDENT schema = (Suppress('schema:') + INDENT + dict_def + UNDENT)\ .setParseAction(lambda toks: toks[0])\ .setResultsName('schema') body = Optional(field_def) & Optional(checker) & Optional(json_extra) \ & Optional(description) & Optional(producer) & Optional(schema) comment = Literal("#") + restOfLine + LineEnd() include = (Suppress("include") + quotedString)\ .setResultsName("includes", listAllMatches=True) rule = (Optional(persistent_identifier) + Optional(inherit_from) + \ Optional(override) + Optional(extend) +json_id + \ Optional(Suppress(",") + aliases) + Suppress(":") + \ INDENT + body + UNDENT)\ .setResultsName("rules", listAllMatches=True) return OneOrMore(rule | include | comment.suppress())
curCol = col(l,s) if curCol > indentStack[-1]: indentStack.append( curCol ) else: raise ParseException(s,l,"not a subentry") def checkUnindent(s,l,t): if l >= len(s): return curCol = col(l,s) if not(curCol < indentStack[-1] and curCol <= indentStack[-2]): raise ParseException(s,l,"not an unindent") def doUnindent(): indentStack.pop() INDENT = lineEnd.suppress() + empty + empty.copy().setParseAction(checkSubIndent) UNDENT = FollowedBy(empty).setParseAction(checkUnindent) UNDENT.setParseAction(doUnindent) stmt = Forward() suite = Group( OneOrMore( empty + stmt.setParseAction( checkPeerIndent ) ) ) identifier = Word(alphas, alphanums) funcDecl = ("def" + identifier + Group( "(" + Optional( delimitedList(identifier) ) + ")" ) + ":") funcDef = Group( funcDecl + INDENT + suite + UNDENT ) rvalue = Forward() funcCall = Group(identifier + "(" + Optional(delimitedList(rvalue)) + ")") rvalue << (funcCall | identifier | Word(nums)) assignment = Group(identifier + "=" + rvalue) stmt << ( funcDef | assignment | identifier )
def _create_config_parser(): """ Creates a parser using pyparsing that works with bibfield rule definitions BNF like grammar: rule ::= ([persitent_identifier] json_id ["[0]" | "[n]"] "," aliases":" INDENT body UNDENT) | include include ::= "include(" PATH ")" body ::= [inherit_from] (creator | derived | calculated) [checker] [documentation] aliases ::= json_id ["[0]" | "[n]"] ["," aliases] creator ::= "creator:" INDENT creator_body+ UNDENT creator_body ::= [parse_first] [legacy] source_format "," source_tag "," python_allowed_expr source_format ::= MASTER_FORMATS source_tag ::= QUOTED_STRING derived ::= "derived" INDENT derived_calculated_body UNDENT calculated ::= "calculated:" INDENT derived_calculated_body UNDENT derived_calculated_body ::= [parse_first] [depends_on] [only_if] [do_not_cache] "," python_allowed_exp peristent_identfier ::= @persitent_identifier( level ) inherit_from ::= "@inherit_from()" legacy ::= "@legacy(" correspondences+ ")" do_not_cache ::= "@do_not_cache" correspondences ::= "(" source_tag [ "," tag_name ] "," json_id ")" parse_first ::= "@parse_first(" jsonid+ ")" depends_on ::= "@depends_on(" json_id+ ")" only_if ::= "@only_if(" python_condition+ ")" python_allowed_exp ::= ident | list_def | dict_def | list_access | dict_access | function_call checker ::= "checker:" INDENT checker_function+ UNDENT documentation ::= INDENT doc_string subfield* UNDENT doc_string ::= QUOTED_STRING subfield ::= "@subfield" json_id["."json_id*] ":" docstring """ indent_stack = [1] def check_sub_indent(str, location, tokens): cur_col = col(location, str) if cur_col > indent_stack[-1]: indent_stack.append(cur_col) else: raise ParseException(str, location, "not a subentry") def check_unindent(str, location, tokens): if location >= len(str): return cur_col = col(location, str) if not(cur_col < indent_stack[-1] and cur_col <= indent_stack[-2]): raise ParseException(str, location, "not an unindent") def do_unindent(): indent_stack.pop() INDENT = lineEnd.suppress() + empty + empty.copy().setParseAction(check_sub_indent) UNDENT = FollowedBy(empty).setParseAction(check_unindent) UNDENT.setParseAction(do_unindent) json_id = (Word(alphanums + "_") + Optional(oneOf("[0] [n]")))\ .setResultsName("json_id", listAllMatches=True)\ .setParseAction(lambda tokens: "".join(tokens)) aliases = delimitedList((Word(alphanums + "_") + Optional(oneOf("[0] [n]"))) .setParseAction(lambda tokens: "".join(tokens)))\ .setResultsName("aliases") python_allowed_expr = Forward() ident = Word(alphas + "_", alphanums + "_") dict_def = originalTextFor(nestedExpr('{', '}')) list_def = originalTextFor(nestedExpr('[', ']')) dict_access = list_access = originalTextFor(ident + nestedExpr('[', ']')) function_call = originalTextFor(ZeroOrMore(ident + ".") + ident + nestedExpr('(', ')')) python_allowed_expr << (ident ^ dict_def ^ list_def ^ dict_access ^ list_access ^ function_call)\ .setResultsName("value", listAllMatches=True) persistent_identifier = (Suppress("@persistent_identifier") + nestedExpr("(", ")"))\ .setResultsName("persistent_identifier") inherit_from = (Suppress("@inherit_from") + originalTextFor(nestedExpr("(", ")")))\ .setResultsName("inherit_from") legacy = (Suppress("@legacy") + originalTextFor(nestedExpr("(", ")")))\ .setResultsName("legacy", listAllMatches=True) only_if = (Suppress("@only_if") + originalTextFor(nestedExpr("(", ")")))\ .setResultsName("only_if") depends_on = (Suppress("@depends_on") + originalTextFor(nestedExpr("(", ")")))\ .setResultsName("depends_on") parse_first = (Suppress("@parse_first") + originalTextFor(nestedExpr("(", ")")))\ .setResultsName("parse_first") do_not_cache = (Suppress("@") + "do_not_cache")\ .setResultsName("do_not_cache") master_format = (Suppress("@master_format") + originalTextFor(nestedExpr("(", ")")))\ .setResultsName("master_format") derived_calculated_body = Optional(parse_first) + Optional(depends_on) + Optional(only_if) + Optional(do_not_cache) + python_allowed_expr derived = "derived" + Suppress(":") + INDENT + derived_calculated_body + UNDENT calculated = "calculated" + Suppress(":") + INDENT + derived_calculated_body + UNDENT source_tag = quotedString\ .setParseAction(removeQuotes)\ .setResultsName("source_tag", listAllMatches=True) source_format = oneOf(CFG_BIBFIELD_MASTER_FORMATS)\ .setResultsName("source_format", listAllMatches=True) creator_body = (Optional(parse_first) + Optional(depends_on) + Optional(only_if) + Optional(legacy) + source_format + Suppress(",") + source_tag + Suppress(",") + python_allowed_expr)\ .setResultsName("creator_def", listAllMatches=True) creator = "creator" + Suppress(":") + INDENT + OneOrMore(creator_body) + UNDENT checker_function = (Optional(master_format) + ZeroOrMore(ident + ".") + ident + originalTextFor(nestedExpr('(', ')')))\ .setResultsName("checker_function", listAllMatches=True) checker = ("checker" + Suppress(":") + INDENT + OneOrMore(checker_function) + UNDENT) doc_string = QuotedString(quoteChar='"""', multiline=True) | quotedString.setParseAction(removeQuotes) subfield = (Suppress("@subfield") + Word(alphanums + "_" + '.') + Suppress(":") + Optional(doc_string))\ .setResultsName("subfields", listAllMatches=True) documentation = ("documentation" + Suppress(":") + INDENT + Optional(doc_string).setResultsName("main_doc") + ZeroOrMore(subfield) + UNDENT)\ .setResultsName("documentation") field_def = (creator | derived | calculated)\ .setResultsName("type_field", listAllMatches=True) body = Optional(inherit_from) + Optional(field_def) + Optional(checker) + Optional(documentation) comment = Literal("#") + restOfLine + LineEnd() include = (Suppress("include") + quotedString)\ .setResultsName("includes", listAllMatches=True) rule = (Optional(persistent_identifier) + json_id + Optional(Suppress(",") + aliases) + Suppress(":") + INDENT + body + UNDENT)\ .setResultsName("rules", listAllMatches=True) return OneOrMore(rule | include | comment.suppress())
def _create_config_parser(): """ Creates a parser using pyparsing that works with bibfield rule definitions BNF like grammar: rule ::= ([persitent_identifier] json_id ["[0]" | "[n]"] "," aliases":" INDENT body UNDENT) | include include ::= "include(" PATH ")" body ::= [inherit_from] (creator | derived | calculated) [checker] [documentation] aliases ::= json_id ["[0]" | "[n]"] ["," aliases] creator ::= "creator:" INDENT creator_body+ UNDENT creator_body ::= [parse_first] [legacy] source_format "," source_tag "," python_allowed_expr source_format ::= MASTER_FORMATS source_tag ::= QUOTED_STRING derived ::= "derived" INDENT derived_calculated_body UNDENT calculated ::= "calculated:" INDENT derived_calculated_body UNDENT derived_calculated_body ::= [parse_first] [depends_on] [only_if] [do_not_cache] "," python_allowed_exp peristent_identfier ::= @persitent_identifier( level ) inherit_from ::= "@inherit_from()" legacy ::= "@legacy(" correspondences+ ")" do_not_cache ::= "@do_not_cache" correspondences ::= "(" source_tag [ "," tag_name ] "," json_id ")" parse_first ::= "@parse_first(" jsonid+ ")" depends_on ::= "@depends_on(" json_id+ ")" only_if ::= "@only_if(" python_condition+ ")" python_allowed_exp ::= ident | list_def | dict_def | list_access | dict_access | function_call checker ::= "checker:" INDENT checker_function+ UNDENT documentation ::= INDENT doc_string subfield* UNDENT doc_string ::= QUOTED_STRING subfield ::= "@subfield" json_id["."json_id*] ":" docstring """ indent_stack = [1] def check_sub_indent(str, location, tokens): cur_col = col(location, str) if cur_col > indent_stack[-1]: indent_stack.append(cur_col) else: raise ParseException(str, location, "not a subentry") def check_unindent(str, location, tokens): if location >= len(str): return cur_col = col(location, str) if not (cur_col < indent_stack[-1] and cur_col <= indent_stack[-2]): raise ParseException(str, location, "not an unindent") def do_unindent(): indent_stack.pop() INDENT = lineEnd.suppress() + empty + empty.copy().setParseAction( check_sub_indent) UNDENT = FollowedBy(empty).setParseAction(check_unindent) UNDENT.setParseAction(do_unindent) json_id = (Word(alphanums + "_") + Optional(oneOf("[0] [n]")))\ .setResultsName("json_id", listAllMatches=True)\ .setParseAction(lambda tokens: "".join(tokens)) aliases = delimitedList((Word(alphanums + "_") + Optional(oneOf("[0] [n]"))) .setParseAction(lambda tokens: "".join(tokens)))\ .setResultsName("aliases") python_allowed_expr = Forward() ident = Word(alphas + "_", alphanums + "_") dict_def = originalTextFor(nestedExpr('{', '}')) list_def = originalTextFor(nestedExpr('[', ']')) dict_access = list_access = originalTextFor(ident + nestedExpr('[', ']')) function_call = originalTextFor( ZeroOrMore(ident + ".") + ident + nestedExpr('(', ')')) python_allowed_expr << (ident ^ dict_def ^ list_def ^ dict_access ^ list_access ^ function_call)\ .setResultsName("value", listAllMatches=True) persistent_identifier = (Suppress("@persistent_identifier") + nestedExpr("(", ")"))\ .setResultsName("persistent_identifier") inherit_from = (Suppress("@inherit_from") + originalTextFor(nestedExpr("(", ")")))\ .setResultsName("inherit_from") legacy = (Suppress("@legacy") + originalTextFor(nestedExpr("(", ")")))\ .setResultsName("legacy", listAllMatches=True) only_if = (Suppress("@only_if") + originalTextFor(nestedExpr("(", ")")))\ .setResultsName("only_if") depends_on = (Suppress("@depends_on") + originalTextFor(nestedExpr("(", ")")))\ .setResultsName("depends_on") parse_first = (Suppress("@parse_first") + originalTextFor(nestedExpr("(", ")")))\ .setResultsName("parse_first") do_not_cache = (Suppress("@") + "do_not_cache")\ .setResultsName("do_not_cache") master_format = (Suppress("@master_format") + originalTextFor(nestedExpr("(", ")")))\ .setResultsName("master_format") derived_calculated_body = Optional(parse_first) + Optional( depends_on) + Optional(only_if) + Optional( do_not_cache) + python_allowed_expr derived = "derived" + Suppress( ":") + INDENT + derived_calculated_body + UNDENT calculated = "calculated" + Suppress( ":") + INDENT + derived_calculated_body + UNDENT source_tag = quotedString\ .setParseAction(removeQuotes)\ .setResultsName("source_tag", listAllMatches=True) source_format = oneOf(CFG_BIBFIELD_MASTER_FORMATS)\ .setResultsName("source_format", listAllMatches=True) creator_body = (Optional(parse_first) + Optional(depends_on) + Optional(only_if) + Optional(legacy) + source_format + Suppress(",") + source_tag + Suppress(",") + python_allowed_expr)\ .setResultsName("creator_def", listAllMatches=True) creator = "creator" + Suppress(":") + INDENT + OneOrMore( creator_body) + UNDENT checker_function = (Optional(master_format) + ZeroOrMore(ident + ".") + ident + originalTextFor(nestedExpr('(', ')')))\ .setResultsName("checker_function", listAllMatches=True) checker = ("checker" + Suppress(":") + INDENT + OneOrMore(checker_function) + UNDENT) doc_string = QuotedString( quoteChar='"""', multiline=True) | quotedString.setParseAction(removeQuotes) subfield = (Suppress("@subfield") + Word(alphanums + "_" + '.') + Suppress(":") + Optional(doc_string))\ .setResultsName("subfields", listAllMatches=True) documentation = ("documentation" + Suppress(":") + INDENT + Optional(doc_string).setResultsName("main_doc") + ZeroOrMore(subfield) + UNDENT)\ .setResultsName("documentation") field_def = (creator | derived | calculated)\ .setResultsName("type_field", listAllMatches=True) body = Optional(inherit_from) + Optional(field_def) + Optional( checker) + Optional(documentation) comment = Literal("#") + restOfLine + LineEnd() include = (Suppress("include") + quotedString)\ .setResultsName("includes", listAllMatches=True) rule = (Optional(persistent_identifier) + json_id + Optional(Suppress(",") + aliases) + Suppress(":") + INDENT + body + UNDENT)\ .setResultsName("rules", listAllMatches=True) return OneOrMore(rule | include | comment.suppress())
not_logic = Literal('not') conditionals = Forward() conditional = Group(comparable_expression + comparison + comparable_expression) | \ Group(lpar + conditionals + rpar) conditionals << Optional(not_logic) + conditional + ZeroOrMore(binary_logic + conditional) if_keyword = Literal('if') if_statement = (if_keyword + conditionals).setParseAction(IfStatement) option = Forward() response_definition = text + lineEnd.suppress() + Optional(if_statement + Literal('then').suppress()) response = (response_definition + ZeroOrMore(indentedBlock(option, indentStack, True))).setParseAction(Response) event_send = Literal('->') event_message_separator = Literal('!').suppress() event_atom = atom.copy().setParseAction(lambda t: repr(t[0])) event_message = quotedString | event_atom event_send_separator = Literal(',').suppress() event_statement = (event_send + event_atom + event_message_separator + event_message).setParseAction(Event) options_delimiter = Literal('~') options_definition = options_delimiter + text + Optional(event_statement + ZeroOrMore(event_send_separator + event_statement)) option << (options_definition + ZeroOrMore(indentedBlock(response, indentStack, True))).setParseAction(Option)
else: raise ParseException(s, l, "not a subentry") def checkUnindent(s, l, t): if l >= len(s): return curCol = col(l, s) if not(curCol < indentStack[-1] and curCol <= indentStack[-2]): raise ParseException(s, l, "not an unindent") def doUnindent(): indentStack.pop() INDENT = lineEnd.suppress() + empty + \ empty.copy().setParseAction(checkSubIndent) UNDENT = FollowedBy(empty).setParseAction(checkUnindent) UNDENT.setParseAction(doUnindent) stmt = Forward() suite = Group(OneOrMore(empty + stmt.setParseAction(checkPeerIndent))) identifier = Word(alphas, alphanums) funcDecl = ("def" + identifier + Group("(" + Optional(delimitedList(identifier)) + ")") + ":") funcDef = Group(funcDecl + INDENT + suite + UNDENT) rvalue = Forward() funcCall = Group(identifier + "(" + Optional(delimitedList(rvalue)) + ")") rvalue << (funcCall | identifier | Word(nums))
def _create_field_parser(): """ Creates a parser using pyparsing that works with bibfield rule definitions BNF like grammar: rule ::= ([persitent_identifier] json_id ["[0]" | "[n]"] "," aliases":" INDENT body UNDENT) | include | python_comment include ::= "include(" PATH ")" body ::= [inherit_from] (creator | derived | calculated) [checker] [documentation] [producer] aliases ::= json_id ["[0]" | "[n]"] ["," aliases] creator ::= "creator:" INDENT creator_body+ UNDENT creator_body ::= [decorators] source_format "," source_tag "," python_allowed_expr source_format ::= MASTER_FORMATS source_tag ::= QUOTED_STRING derived ::= "derived" INDENT derived_calculated_body UNDENT calculated ::= "calculated:" INDENT derived_calculated_body UNDENT derived_calculated_body ::= [decorators] "," python_allowed_exp decorators ::= (peristent_identfier | legacy | do_not_cache | parse_first | depends_on | only_if | only_if_master_value)* peristent_identfier ::= @persitent_identifier( level ) legacy ::= "@legacy(" correspondences+ ")" correspondences ::= "(" source_tag [ "," tag_name ] "," json_id ")" parse_first ::= "@parse_first(" jsonid+ ")" depends_on ::= "@depends_on(" json_id+ ")" only_if ::= "@only_if(" python_condition+ ")" only_if_master_value ::= "@only_if_master_value(" python_condition+ ")" inherit_from ::= "@inherit_from()" python_allowed_exp ::= ident | list_def | dict_def | list_access | dict_access | function_call checker ::= "checker:" INDENT checker_function+ UNDENT documentation ::= INDENT doc_string subfield* UNDENT doc_string ::= QUOTED_STRING subfield ::= "@subfield" json_id["."json_id*] ":" docstring producer ::= "producer:" INDENT producer_body UNDENT producer_body ::= producer_code "," python_dictionary producer_code ::= ident """ indent_stack = [1] def check_sub_indent(str, location, tokens): cur_col = col(location, str) if cur_col > indent_stack[-1]: indent_stack.append(cur_col) else: raise ParseException(str, location, "not a subentry") def check_unindent(str, location, tokens): if location >= len(str): return cur_col = col(location, str) if not (cur_col < indent_stack[-1] and cur_col <= indent_stack[-2]): raise ParseException(str, location, "not an unindent") def do_unindent(): indent_stack.pop() INDENT = lineEnd.suppress() + empty + empty.copy().setParseAction( check_sub_indent) UNDENT = FollowedBy(empty).setParseAction(check_unindent) UNDENT.setParseAction(do_unindent) json_id = (Word(alphas + "_", alphanums + "_") + Optional(oneOf("[0] [n]")))\ .setResultsName("json_id", listAllMatches=True)\ .setParseAction(lambda tokens: "".join(tokens)) aliases = delimitedList((Word(alphanums + "_") + Optional(oneOf("[0] [n]"))) .setParseAction(lambda tokens: "".join(tokens)))\ .setResultsName("aliases") ident = Word(alphas + "_", alphanums + "_") dict_def = originalTextFor(nestedExpr('{', '}')) list_def = originalTextFor(nestedExpr('[', ']')) dict_access = list_access = originalTextFor(ident + nestedExpr('[', ']')) function_call = originalTextFor( ZeroOrMore(ident + ".") + ident + nestedExpr('(', ')')) python_allowed_expr = (dict_def ^ list_def ^ dict_access ^ \ list_access ^ function_call ^ restOfLine)\ .setResultsName("value", listAllMatches=True) persistent_identifier = (Suppress("@persistent_identifier") + \ nestedExpr("(", ")"))\ .setResultsName("persistent_identifier") legacy = (Suppress("@legacy") + originalTextFor(nestedExpr("(", ")")))\ .setResultsName("legacy", listAllMatches=True) only_if = (Suppress("@only_if") + originalTextFor(nestedExpr("(", ")")))\ .setResultsName("only_if") only_if_master_value = (Suppress("@only_if_value") + \ originalTextFor(nestedExpr("(", ")")))\ .setResultsName("only_if_master_value") depends_on = (Suppress("@depends_on") + \ originalTextFor(nestedExpr("(", ")")))\ .setResultsName("depends_on") parse_first = (Suppress("@parse_first") + \ originalTextFor(nestedExpr("(", ")")))\ .setResultsName("parse_first") memoize = (Suppress("@memoize") + nestedExpr("(", ")"))\ .setResultsName("memoize") field_decorator = parse_first ^ depends_on ^ only_if ^ \ only_if_master_value ^ memoize ^ legacy #Independent decorators inherit_from = (Suppress("@inherit_from") + \ originalTextFor(nestedExpr("(", ")")))\ .setResultsName("inherit_from") override = (Suppress("@") + "override")\ .setResultsName("override") extend = (Suppress("@") + "extend")\ .setResultsName("extend") master_format = (Suppress("@master_format") + \ originalTextFor(nestedExpr("(", ")")))\ .setResultsName("master_format") \ .setParseAction(lambda toks: toks[0]) derived_calculated_body = (ZeroOrMore(field_decorator) + python_allowed_expr)\ .setResultsName('derived_calculated_def') derived = "derived" + Suppress(":") + \ INDENT + derived_calculated_body + UNDENT calculated = "calculated" + Suppress(":") + \ INDENT + derived_calculated_body + UNDENT source_tag = quotedString\ .setParseAction(removeQuotes)\ .setResultsName("source_tag", listAllMatches=True) source_format = Word(alphas, alphanums + "_")\ .setResultsName("source_format", listAllMatches=True) creator_body = (ZeroOrMore(field_decorator) + source_format + \ Suppress(",") + source_tag + Suppress(",") + python_allowed_expr)\ .setResultsName("creator_def", listAllMatches=True) creator = "creator" + Suppress(":") + \ INDENT + OneOrMore(creator_body) + UNDENT field_def = (creator | derived | calculated)\ .setResultsName("type_field", listAllMatches=True) #JsonExtra json_dumps = (Suppress('dumps') + Suppress(',') + python_allowed_expr)\ .setResultsName("dumps")\ .setParseAction(lambda toks: toks.value[0]) json_loads = (Suppress("loads") + Suppress(",") + python_allowed_expr)\ .setResultsName("loads")\ .setParseAction(lambda toks: toks.value[0]) json_extra = (Suppress('json:') + \ INDENT + Each((json_dumps, json_loads)) + UNDENT)\ .setResultsName('json_ext') #Checker checker_function = (Optional(master_format) + ZeroOrMore(ident + ".") + ident + originalTextFor(nestedExpr('(', ')')))\ .setResultsName("checker", listAllMatches=True) checker = ("checker" + Suppress(":") + INDENT + OneOrMore(checker_function) + UNDENT) #Description/Documentation doc_double = QuotedString(quoteChar='"""', multiline=True) doc_single = QuotedString(quoteChar="'''", multiline=True) doc_string = INDENT + (doc_double | doc_single) + UNDENT description_body = (Suppress('description:') + doc_string).\ setParseAction(lambda toks: toks[0][0]) description = (description_body | doc_double | doc_single)\ .setResultsName('description') #Producer producer_code = (Word(alphas, alphanums + "_")\ + originalTextFor(nestedExpr("(", ")")))\ .setResultsName('producer_code', listAllMatches=True) producer_body = (producer_code + Suppress(",") + python_allowed_expr)\ .setResultsName("producer_rule", listAllMatches=True) producer = Suppress("producer:") + INDENT + OneOrMore( producer_body) + UNDENT schema = (Suppress('schema:') + INDENT + dict_def + UNDENT)\ .setParseAction(lambda toks: toks[0])\ .setResultsName('schema') body = Optional(field_def) & Optional(checker) & Optional(json_extra) \ & Optional(description) & Optional(producer) & Optional(schema) comment = Literal("#") + restOfLine + LineEnd() include = (Suppress("include") + quotedString)\ .setResultsName("includes", listAllMatches=True) rule = (Optional(persistent_identifier) + Optional(inherit_from) + \ Optional(override) + Optional(extend) +json_id + \ Optional(Suppress(",") + aliases) + Suppress(":") + \ INDENT + body + UNDENT)\ .setResultsName("rules", listAllMatches=True) return OneOrMore(rule | include | comment.suppress())