def indentedBlock(expr, indent_stack, indent=True):
"""Define space-delimited indentation blocks.
Helper method for defining space-delimited indentation blocks, such as
those used to define block statements in Python source code.
There is also a version in pyparsing but doesn't seem to be working fine
with JSONAlchemy cfg files.
"""
def check_sub_indent(string, location, tokens):
"""Check the indentation."""
cur_col = col(location, string)
if cur_col > indent_stack[-1]:
indent_stack.append(cur_col)
else:
raise ParseException(string, location, "not a subentry")
def check_unindent(string, location, tokens):
"""Check the 'undentation'."""
if location >= len(string):
return
cur_col = col(location, string)
if not(cur_col < indent_stack[-1] and cur_col <= indent_stack[-2]):
raise ParseException(string, location, "not an unindent")
def do_unindent():
"""Unindent."""
indent_stack.pop()
indent = lineEnd.suppress() + empty + empty.copy()\
.setParseAction(check_sub_indent)
undent = FollowedBy(empty).setParseAction(check_unindent)
undent.setParseAction(do_unindent)
return indent + expr + undent
def indentedBlock(expr, indent_stack, indent=True):
"""Define space-delimited indentation blocks.
Helper method for defining space-delimited indentation blocks, such as
those used to define block statements in Python source code.
There is also a version in pyparsing but doesn't seem to be working fine
with JSONAlchemy cfg files.
"""
def check_sub_indent(string, location, tokens):
"""Check the indentation."""
cur_col = col(location, string)
if cur_col > indent_stack[-1]:
indent_stack.append(cur_col)
else:
raise ParseException(string, location, "not a subentry")
def check_unindent(string, location, tokens):
"""Check the 'undentation'."""
if location >= len(string):
return
cur_col = col(location, string)
if not (cur_col < indent_stack[-1] and cur_col <= indent_stack[-2]):
raise ParseException(string, location, "not an unindent")
def do_unindent():
"""Unindent."""
indent_stack.pop()
indent = lineEnd.suppress() + empty + empty.copy()\
.setParseAction(check_sub_indent)
undent = FollowedBy(empty).setParseAction(check_unindent)
undent.setParseAction(do_unindent)
return indent + expr + undent
def parse_pabl(self, raw_pabl):
INDENT = lineEnd.suppress() + empty + empty.copy().setParseAction(
self.check_sub_indent)
UNDENT = FollowedBy(empty).setParseAction(self.check_unindent)
UNDENT.setParseAction(self.unindent)
terminator = Literal(';').suppress()
comment = Literal('#') + restOfLine
item_name = Word(alphas, alphanums + '_')
variable = Word(alphas, alphanums + '_.')
variable_as = (variable + 'as' + item_name)
stmt = Forward()
suite = Group(
OneOrMore(empty + stmt.setParseAction(self.check_peer_indent)))
suite.ignore(comment)
item_start = Literal('@item').suppress()
item_end = Literal(':').suppress()
permission_start = Literal('@permissions')
item_decl = (item_start + item_name.setResultsName('item') + item_end)
item_defn = Group(item_decl + INDENT + suite + UNDENT)
permission_decl = (permission_start + Group(
delimitedList(item_name).setResultsName('permissions')) + item_end)
permission_defn = Group(permission_decl + INDENT + suite + UNDENT)
fieldList = delimitedList(
Group(variable_as) | variable
).setResultsName('fields') + terminator
stmt << (item_defn | fieldList | Group(permission_defn))
parseTree = suite.parseString(raw_pabl)
return parseTree
def _create_field_parser():
"""
Creates a parser using pyparsing that works with bibfield rule definitions
BNF like grammar:
rule ::= ([persitent_identifier] json_id ["[0]" | "[n]"] "," aliases":" INDENT body UNDENT) | include | python_comment
include ::= "include(" PATH ")"
body ::= [inherit_from] (creator | derived | calculated) [checker] [documentation] [producer]
aliases ::= json_id ["[0]" | "[n]"] ["," aliases]
creator ::= "creator:" INDENT creator_body+ UNDENT
creator_body ::= [decorators] source_format "," source_tag "," python_allowed_expr
source_format ::= MASTER_FORMATS
source_tag ::= QUOTED_STRING
derived ::= "derived" INDENT derived_calculated_body UNDENT
calculated ::= "calculated:" INDENT derived_calculated_body UNDENT
derived_calculated_body ::= [decorators] "," python_allowed_exp
decorators ::= (peristent_identfier | legacy | do_not_cache | parse_first | depends_on | only_if | only_if_master_value)*
peristent_identfier ::= @persitent_identifier( level )
legacy ::= "@legacy(" correspondences+ ")"
correspondences ::= "(" source_tag [ "," tag_name ] "," json_id ")"
parse_first ::= "@parse_first(" jsonid+ ")"
depends_on ::= "@depends_on(" json_id+ ")"
only_if ::= "@only_if(" python_condition+ ")"
only_if_master_value ::= "@only_if_master_value(" python_condition+ ")"
inherit_from ::= "@inherit_from()"
python_allowed_exp ::= ident | list_def | dict_def | list_access | dict_access | function_call
checker ::= "checker:" INDENT checker_function+ UNDENT
documentation ::= INDENT doc_string subfield* UNDENT
doc_string ::= QUOTED_STRING
subfield ::= "@subfield" json_id["."json_id*] ":" docstring
producer ::= "producer:" INDENT producer_body UNDENT
producer_body ::= producer_code "," python_dictionary
producer_code ::= ident
"""
indent_stack = [1]
def check_sub_indent(str, location, tokens):
cur_col = col(location, str)
if cur_col > indent_stack[-1]:
indent_stack.append(cur_col)
else:
raise ParseException(str, location, "not a subentry")
def check_unindent(str, location, tokens):
if location >= len(str):
return
cur_col = col(location, str)
if not(cur_col < indent_stack[-1] and cur_col <= indent_stack[-2]):
raise ParseException(str, location, "not an unindent")
def do_unindent():
indent_stack.pop()
INDENT = lineEnd.suppress() + empty + empty.copy().setParseAction(check_sub_indent)
UNDENT = FollowedBy(empty).setParseAction(check_unindent)
UNDENT.setParseAction(do_unindent)
json_id = (Word(alphas + "_", alphanums + "_") + Optional(oneOf("[0] [n]")))\
.setResultsName("json_id", listAllMatches=True)\
.setParseAction(lambda tokens: "".join(tokens))
aliases = delimitedList((Word(alphanums + "_") + Optional(oneOf("[0] [n]")))
.setParseAction(lambda tokens: "".join(tokens)))\
.setResultsName("aliases")
ident = Word(alphas + "_", alphanums + "_")
dict_def = originalTextFor(nestedExpr('{', '}'))
list_def = originalTextFor(nestedExpr('[', ']'))
dict_access = list_access = originalTextFor(ident + nestedExpr('[', ']'))
function_call = originalTextFor(ZeroOrMore(ident + ".") + ident + nestedExpr('(', ')'))
python_allowed_expr = (dict_def ^ list_def ^ dict_access ^ \
list_access ^ function_call ^ restOfLine)\
.setResultsName("value", listAllMatches=True)
persistent_identifier = (Suppress("@persistent_identifier") + \
nestedExpr("(", ")"))\
.setResultsName("persistent_identifier")
legacy = (Suppress("@legacy") + originalTextFor(nestedExpr("(", ")")))\
.setResultsName("legacy", listAllMatches=True)
only_if = (Suppress("@only_if") + originalTextFor(nestedExpr("(", ")")))\
.setResultsName("only_if")
only_if_master_value = (Suppress("@only_if_value") + \
originalTextFor(nestedExpr("(", ")")))\
.setResultsName("only_if_master_value")
depends_on = (Suppress("@depends_on") + \
originalTextFor(nestedExpr("(", ")")))\
.setResultsName("depends_on")
parse_first = (Suppress("@parse_first") + \
originalTextFor(nestedExpr("(", ")")))\
.setResultsName("parse_first")
memoize = (Suppress("@memoize") + nestedExpr("(", ")"))\
.setResultsName("memoize")
field_decorator = parse_first ^ depends_on ^ only_if ^ \
only_if_master_value ^ memoize ^ legacy
#Independent decorators
inherit_from = (Suppress("@inherit_from") + \
originalTextFor(nestedExpr("(", ")")))\
.setResultsName("inherit_from")
override = (Suppress("@") + "override")\
.setResultsName("override")
extend = (Suppress("@") + "extend")\
.setResultsName("extend")
master_format = (Suppress("@master_format") + \
originalTextFor(nestedExpr("(", ")")))\
.setResultsName("master_format") \
.setParseAction(lambda toks: toks[0])
derived_calculated_body = (ZeroOrMore(field_decorator) + python_allowed_expr)\
.setResultsName('derived_calculated_def')
derived = "derived" + Suppress(":") + \
INDENT + derived_calculated_body + UNDENT
calculated = "calculated" + Suppress(":") + \
INDENT + derived_calculated_body + UNDENT
source_tag = quotedString\
.setParseAction(removeQuotes)\
.setResultsName("source_tag", listAllMatches=True)
source_format = Word(alphas, alphanums + "_")\
.setResultsName("source_format", listAllMatches=True)
creator_body = (ZeroOrMore(field_decorator) + source_format + \
Suppress(",") + source_tag + Suppress(",") + python_allowed_expr)\
.setResultsName("creator_def", listAllMatches=True)
creator = "creator" + Suppress(":") + \
INDENT + OneOrMore(creator_body) + UNDENT
field_def = (creator | derived | calculated)\
.setResultsName("type_field", listAllMatches=True)
#JsonExtra
json_dumps = (Suppress('dumps') + Suppress(',') + python_allowed_expr)\
.setResultsName("dumps")\
.setParseAction(lambda toks: toks.value[0])
json_loads = (Suppress("loads") + Suppress(",") + python_allowed_expr)\
.setResultsName("loads")\
.setParseAction(lambda toks: toks.value[0])
json_extra = (Suppress('json:') + \
INDENT + Each((json_dumps, json_loads)) + UNDENT)\
.setResultsName('json_ext')
#Checker
checker_function = (Optional(master_format) + ZeroOrMore(ident + ".") + ident + originalTextFor(nestedExpr('(', ')')))\
.setResultsName("checker", listAllMatches=True)
checker = ("checker" + Suppress(":") + INDENT + OneOrMore(checker_function) + UNDENT)
#Description/Documentation
doc_double = QuotedString(quoteChar='"""', multiline=True)
doc_single = QuotedString(quoteChar="'''", multiline=True)
doc_string = INDENT + (doc_double | doc_single) + UNDENT
description_body = (Suppress('description:') + doc_string).\
setParseAction(lambda toks: toks[0][0])
description = (description_body | doc_double | doc_single)\
.setResultsName('description')
#Producer
producer_code = (Word(alphas, alphanums + "_")\
+ originalTextFor(nestedExpr("(", ")")))\
.setResultsName('producer_code', listAllMatches=True)
producer_body = (producer_code + Suppress(",") + python_allowed_expr)\
.setResultsName("producer_rule", listAllMatches=True)
producer = Suppress("producer:") + INDENT + OneOrMore(producer_body) + UNDENT
schema = (Suppress('schema:') + INDENT + dict_def + UNDENT)\
.setParseAction(lambda toks: toks[0])\
.setResultsName('schema')
body = Optional(field_def) & Optional(checker) & Optional(json_extra) \
& Optional(description) & Optional(producer) & Optional(schema)
comment = Literal("#") + restOfLine + LineEnd()
include = (Suppress("include") + quotedString)\
.setResultsName("includes", listAllMatches=True)
rule = (Optional(persistent_identifier) + Optional(inherit_from) + \
Optional(override) + Optional(extend) +json_id + \
Optional(Suppress(",") + aliases) + Suppress(":") + \
INDENT + body + UNDENT)\
.setResultsName("rules", listAllMatches=True)
return OneOrMore(rule | include | comment.suppress())
curCol = col(l,s)
if curCol > indentStack[-1]:
indentStack.append( curCol )
else:
raise ParseException(s,l,"not a subentry")
def checkUnindent(s,l,t):
if l >= len(s): return
curCol = col(l,s)
if not(curCol < indentStack[-1] and curCol <= indentStack[-2]):
raise ParseException(s,l,"not an unindent")
def doUnindent():
indentStack.pop()
INDENT = lineEnd.suppress() + empty + empty.copy().setParseAction(checkSubIndent)
UNDENT = FollowedBy(empty).setParseAction(checkUnindent)
UNDENT.setParseAction(doUnindent)
stmt = Forward()
suite = Group( OneOrMore( empty + stmt.setParseAction( checkPeerIndent ) ) )
identifier = Word(alphas, alphanums)
funcDecl = ("def" + identifier + Group( "(" + Optional( delimitedList(identifier) ) + ")" ) + ":")
funcDef = Group( funcDecl + INDENT + suite + UNDENT )
rvalue = Forward()
funcCall = Group(identifier + "(" + Optional(delimitedList(rvalue)) + ")")
rvalue << (funcCall | identifier | Word(nums))
assignment = Group(identifier + "=" + rvalue)
stmt << ( funcDef | assignment | identifier )
def _create_config_parser():
"""
Creates a parser using pyparsing that works with bibfield rule definitions
BNF like grammar:
rule ::= ([persitent_identifier] json_id ["[0]" | "[n]"] "," aliases":" INDENT body UNDENT) | include
include ::= "include(" PATH ")"
body ::= [inherit_from] (creator | derived | calculated) [checker] [documentation]
aliases ::= json_id ["[0]" | "[n]"] ["," aliases]
creator ::= "creator:" INDENT creator_body+ UNDENT
creator_body ::= [parse_first] [legacy] source_format "," source_tag "," python_allowed_expr
source_format ::= MASTER_FORMATS
source_tag ::= QUOTED_STRING
derived ::= "derived" INDENT derived_calculated_body UNDENT
calculated ::= "calculated:" INDENT derived_calculated_body UNDENT
derived_calculated_body ::= [parse_first] [depends_on] [only_if] [do_not_cache] "," python_allowed_exp
peristent_identfier ::= @persitent_identifier( level )
inherit_from ::= "@inherit_from()"
legacy ::= "@legacy(" correspondences+ ")"
do_not_cache ::= "@do_not_cache"
correspondences ::= "(" source_tag [ "," tag_name ] "," json_id ")"
parse_first ::= "@parse_first(" jsonid+ ")"
depends_on ::= "@depends_on(" json_id+ ")"
only_if ::= "@only_if(" python_condition+ ")"
python_allowed_exp ::= ident | list_def | dict_def | list_access | dict_access | function_call
checker ::= "checker:" INDENT checker_function+ UNDENT
documentation ::= INDENT doc_string subfield* UNDENT
doc_string ::= QUOTED_STRING
subfield ::= "@subfield" json_id["."json_id*] ":" docstring
"""
indent_stack = [1]
def check_sub_indent(str, location, tokens):
cur_col = col(location, str)
if cur_col > indent_stack[-1]:
indent_stack.append(cur_col)
else:
raise ParseException(str, location, "not a subentry")
def check_unindent(str, location, tokens):
if location >= len(str):
return
cur_col = col(location, str)
if not(cur_col < indent_stack[-1] and cur_col <= indent_stack[-2]):
raise ParseException(str, location, "not an unindent")
def do_unindent():
indent_stack.pop()
INDENT = lineEnd.suppress() + empty + empty.copy().setParseAction(check_sub_indent)
UNDENT = FollowedBy(empty).setParseAction(check_unindent)
UNDENT.setParseAction(do_unindent)
json_id = (Word(alphanums + "_") + Optional(oneOf("[0] [n]")))\
.setResultsName("json_id", listAllMatches=True)\
.setParseAction(lambda tokens: "".join(tokens))
aliases = delimitedList((Word(alphanums + "_") + Optional(oneOf("[0] [n]")))
.setParseAction(lambda tokens: "".join(tokens)))\
.setResultsName("aliases")
python_allowed_expr = Forward()
ident = Word(alphas + "_", alphanums + "_")
dict_def = originalTextFor(nestedExpr('{', '}'))
list_def = originalTextFor(nestedExpr('[', ']'))
dict_access = list_access = originalTextFor(ident + nestedExpr('[', ']'))
function_call = originalTextFor(ZeroOrMore(ident + ".") + ident + nestedExpr('(', ')'))
python_allowed_expr << (ident ^ dict_def ^ list_def ^ dict_access ^ list_access ^ function_call)\
.setResultsName("value", listAllMatches=True)
persistent_identifier = (Suppress("@persistent_identifier") + nestedExpr("(", ")"))\
.setResultsName("persistent_identifier")
inherit_from = (Suppress("@inherit_from") + originalTextFor(nestedExpr("(", ")")))\
.setResultsName("inherit_from")
legacy = (Suppress("@legacy") + originalTextFor(nestedExpr("(", ")")))\
.setResultsName("legacy", listAllMatches=True)
only_if = (Suppress("@only_if") + originalTextFor(nestedExpr("(", ")")))\
.setResultsName("only_if")
depends_on = (Suppress("@depends_on") + originalTextFor(nestedExpr("(", ")")))\
.setResultsName("depends_on")
parse_first = (Suppress("@parse_first") + originalTextFor(nestedExpr("(", ")")))\
.setResultsName("parse_first")
do_not_cache = (Suppress("@") + "do_not_cache")\
.setResultsName("do_not_cache")
master_format = (Suppress("@master_format") + originalTextFor(nestedExpr("(", ")")))\
.setResultsName("master_format")
derived_calculated_body = Optional(parse_first) + Optional(depends_on) + Optional(only_if) + Optional(do_not_cache) + python_allowed_expr
derived = "derived" + Suppress(":") + INDENT + derived_calculated_body + UNDENT
calculated = "calculated" + Suppress(":") + INDENT + derived_calculated_body + UNDENT
source_tag = quotedString\
.setParseAction(removeQuotes)\
.setResultsName("source_tag", listAllMatches=True)
source_format = oneOf(CFG_BIBFIELD_MASTER_FORMATS)\
.setResultsName("source_format", listAllMatches=True)
creator_body = (Optional(parse_first) + Optional(depends_on) + Optional(only_if) + Optional(legacy) + source_format + Suppress(",") + source_tag + Suppress(",") + python_allowed_expr)\
.setResultsName("creator_def", listAllMatches=True)
creator = "creator" + Suppress(":") + INDENT + OneOrMore(creator_body) + UNDENT
checker_function = (Optional(master_format) + ZeroOrMore(ident + ".") + ident + originalTextFor(nestedExpr('(', ')')))\
.setResultsName("checker_function", listAllMatches=True)
checker = ("checker" + Suppress(":") + INDENT + OneOrMore(checker_function) + UNDENT)
doc_string = QuotedString(quoteChar='"""', multiline=True) | quotedString.setParseAction(removeQuotes)
subfield = (Suppress("@subfield") + Word(alphanums + "_" + '.') + Suppress(":") + Optional(doc_string))\
.setResultsName("subfields", listAllMatches=True)
documentation = ("documentation" + Suppress(":") + INDENT + Optional(doc_string).setResultsName("main_doc") + ZeroOrMore(subfield) + UNDENT)\
.setResultsName("documentation")
field_def = (creator | derived | calculated)\
.setResultsName("type_field", listAllMatches=True)
body = Optional(inherit_from) + Optional(field_def) + Optional(checker) + Optional(documentation)
comment = Literal("#") + restOfLine + LineEnd()
include = (Suppress("include") + quotedString)\
.setResultsName("includes", listAllMatches=True)
rule = (Optional(persistent_identifier) + json_id + Optional(Suppress(",") + aliases) + Suppress(":") + INDENT + body + UNDENT)\
.setResultsName("rules", listAllMatches=True)
return OneOrMore(rule | include | comment.suppress())
def _create_config_parser():
"""
Creates a parser using pyparsing that works with bibfield rule definitions
BNF like grammar:
rule ::= ([persitent_identifier] json_id ["[0]" | "[n]"] "," aliases":" INDENT body UNDENT) | include
include ::= "include(" PATH ")"
body ::= [inherit_from] (creator | derived | calculated) [checker] [documentation]
aliases ::= json_id ["[0]" | "[n]"] ["," aliases]
creator ::= "creator:" INDENT creator_body+ UNDENT
creator_body ::= [parse_first] [legacy] source_format "," source_tag "," python_allowed_expr
source_format ::= MASTER_FORMATS
source_tag ::= QUOTED_STRING
derived ::= "derived" INDENT derived_calculated_body UNDENT
calculated ::= "calculated:" INDENT derived_calculated_body UNDENT
derived_calculated_body ::= [parse_first] [depends_on] [only_if] [do_not_cache] "," python_allowed_exp
peristent_identfier ::= @persitent_identifier( level )
inherit_from ::= "@inherit_from()"
legacy ::= "@legacy(" correspondences+ ")"
do_not_cache ::= "@do_not_cache"
correspondences ::= "(" source_tag [ "," tag_name ] "," json_id ")"
parse_first ::= "@parse_first(" jsonid+ ")"
depends_on ::= "@depends_on(" json_id+ ")"
only_if ::= "@only_if(" python_condition+ ")"
python_allowed_exp ::= ident | list_def | dict_def | list_access | dict_access | function_call
checker ::= "checker:" INDENT checker_function+ UNDENT
documentation ::= INDENT doc_string subfield* UNDENT
doc_string ::= QUOTED_STRING
subfield ::= "@subfield" json_id["."json_id*] ":" docstring
"""
indent_stack = [1]
def check_sub_indent(str, location, tokens):
cur_col = col(location, str)
if cur_col > indent_stack[-1]:
indent_stack.append(cur_col)
else:
raise ParseException(str, location, "not a subentry")
def check_unindent(str, location, tokens):
if location >= len(str):
return
cur_col = col(location, str)
if not (cur_col < indent_stack[-1] and cur_col <= indent_stack[-2]):
raise ParseException(str, location, "not an unindent")
def do_unindent():
indent_stack.pop()
INDENT = lineEnd.suppress() + empty + empty.copy().setParseAction(
check_sub_indent)
UNDENT = FollowedBy(empty).setParseAction(check_unindent)
UNDENT.setParseAction(do_unindent)
json_id = (Word(alphanums + "_") + Optional(oneOf("[0] [n]")))\
.setResultsName("json_id", listAllMatches=True)\
.setParseAction(lambda tokens: "".join(tokens))
aliases = delimitedList((Word(alphanums + "_") + Optional(oneOf("[0] [n]")))
.setParseAction(lambda tokens: "".join(tokens)))\
.setResultsName("aliases")
python_allowed_expr = Forward()
ident = Word(alphas + "_", alphanums + "_")
dict_def = originalTextFor(nestedExpr('{', '}'))
list_def = originalTextFor(nestedExpr('[', ']'))
dict_access = list_access = originalTextFor(ident + nestedExpr('[', ']'))
function_call = originalTextFor(
ZeroOrMore(ident + ".") + ident + nestedExpr('(', ')'))
python_allowed_expr << (ident ^ dict_def ^ list_def ^ dict_access ^ list_access ^ function_call)\
.setResultsName("value", listAllMatches=True)
persistent_identifier = (Suppress("@persistent_identifier") + nestedExpr("(", ")"))\
.setResultsName("persistent_identifier")
inherit_from = (Suppress("@inherit_from") + originalTextFor(nestedExpr("(", ")")))\
.setResultsName("inherit_from")
legacy = (Suppress("@legacy") + originalTextFor(nestedExpr("(", ")")))\
.setResultsName("legacy", listAllMatches=True)
only_if = (Suppress("@only_if") + originalTextFor(nestedExpr("(", ")")))\
.setResultsName("only_if")
depends_on = (Suppress("@depends_on") + originalTextFor(nestedExpr("(", ")")))\
.setResultsName("depends_on")
parse_first = (Suppress("@parse_first") + originalTextFor(nestedExpr("(", ")")))\
.setResultsName("parse_first")
do_not_cache = (Suppress("@") + "do_not_cache")\
.setResultsName("do_not_cache")
master_format = (Suppress("@master_format") + originalTextFor(nestedExpr("(", ")")))\
.setResultsName("master_format")
derived_calculated_body = Optional(parse_first) + Optional(
depends_on) + Optional(only_if) + Optional(
do_not_cache) + python_allowed_expr
derived = "derived" + Suppress(
":") + INDENT + derived_calculated_body + UNDENT
calculated = "calculated" + Suppress(
":") + INDENT + derived_calculated_body + UNDENT
source_tag = quotedString\
.setParseAction(removeQuotes)\
.setResultsName("source_tag", listAllMatches=True)
source_format = oneOf(CFG_BIBFIELD_MASTER_FORMATS)\
.setResultsName("source_format", listAllMatches=True)
creator_body = (Optional(parse_first) + Optional(depends_on) + Optional(only_if) + Optional(legacy) + source_format + Suppress(",") + source_tag + Suppress(",") + python_allowed_expr)\
.setResultsName("creator_def", listAllMatches=True)
creator = "creator" + Suppress(":") + INDENT + OneOrMore(
creator_body) + UNDENT
checker_function = (Optional(master_format) + ZeroOrMore(ident + ".") + ident + originalTextFor(nestedExpr('(', ')')))\
.setResultsName("checker_function", listAllMatches=True)
checker = ("checker" + Suppress(":") + INDENT +
OneOrMore(checker_function) + UNDENT)
doc_string = QuotedString(
quoteChar='"""',
multiline=True) | quotedString.setParseAction(removeQuotes)
subfield = (Suppress("@subfield") + Word(alphanums + "_" + '.') + Suppress(":") + Optional(doc_string))\
.setResultsName("subfields", listAllMatches=True)
documentation = ("documentation" + Suppress(":") + INDENT + Optional(doc_string).setResultsName("main_doc") + ZeroOrMore(subfield) + UNDENT)\
.setResultsName("documentation")
field_def = (creator | derived | calculated)\
.setResultsName("type_field", listAllMatches=True)
body = Optional(inherit_from) + Optional(field_def) + Optional(
checker) + Optional(documentation)
comment = Literal("#") + restOfLine + LineEnd()
include = (Suppress("include") + quotedString)\
.setResultsName("includes", listAllMatches=True)
rule = (Optional(persistent_identifier) + json_id + Optional(Suppress(",") + aliases) + Suppress(":") + INDENT + body + UNDENT)\
.setResultsName("rules", listAllMatches=True)
return OneOrMore(rule | include | comment.suppress())
raise ParseException(s, l, "not a subentry")
def checkUnindent(s, l, t):
if l >= len(s):
return
curCol = col(l, s)
if not(curCol < indentStack[-1] and curCol <= indentStack[-2]):
raise ParseException(s, l, "not an unindent")
def doUnindent():
indentStack.pop()
INDENT = lineEnd.suppress() + empty + \
empty.copy().setParseAction(checkSubIndent)
UNDENT = FollowedBy(empty).setParseAction(checkUnindent)
UNDENT.setParseAction(doUnindent)
stmt = Forward()
suite = Group(OneOrMore(empty + stmt.setParseAction(checkPeerIndent)))
identifier = Word(alphas, alphanums)
funcDecl = ("def" + identifier +
Group("(" + Optional(delimitedList(identifier)) + ")") + ":")
funcDef = Group(funcDecl + INDENT + suite + UNDENT)
rvalue = Forward()
funcCall = Group(identifier + "(" + Optional(delimitedList(rvalue)) + ")")
rvalue << (funcCall | identifier | Word(nums))
assignment = Group(identifier + "=" + rvalue)
def _create_field_parser():
"""
Creates a parser using pyparsing that works with bibfield rule definitions
BNF like grammar:
rule ::= ([persitent_identifier] json_id ["[0]" | "[n]"] "," aliases":" INDENT body UNDENT) | include | python_comment
include ::= "include(" PATH ")"
body ::= [inherit_from] (creator | derived | calculated) [checker] [documentation] [producer]
aliases ::= json_id ["[0]" | "[n]"] ["," aliases]
creator ::= "creator:" INDENT creator_body+ UNDENT
creator_body ::= [decorators] source_format "," source_tag "," python_allowed_expr
source_format ::= MASTER_FORMATS
source_tag ::= QUOTED_STRING
derived ::= "derived" INDENT derived_calculated_body UNDENT
calculated ::= "calculated:" INDENT derived_calculated_body UNDENT
derived_calculated_body ::= [decorators] "," python_allowed_exp
decorators ::= (peristent_identfier | legacy | do_not_cache | parse_first | depends_on | only_if | only_if_master_value)*
peristent_identfier ::= @persitent_identifier( level )
legacy ::= "@legacy(" correspondences+ ")"
correspondences ::= "(" source_tag [ "," tag_name ] "," json_id ")"
parse_first ::= "@parse_first(" jsonid+ ")"
depends_on ::= "@depends_on(" json_id+ ")"
only_if ::= "@only_if(" python_condition+ ")"
only_if_master_value ::= "@only_if_master_value(" python_condition+ ")"
inherit_from ::= "@inherit_from()"
python_allowed_exp ::= ident | list_def | dict_def | list_access | dict_access | function_call
checker ::= "checker:" INDENT checker_function+ UNDENT
documentation ::= INDENT doc_string subfield* UNDENT
doc_string ::= QUOTED_STRING
subfield ::= "@subfield" json_id["."json_id*] ":" docstring
producer ::= "producer:" INDENT producer_body UNDENT
producer_body ::= producer_code "," python_dictionary
producer_code ::= ident
"""
indent_stack = [1]
def check_sub_indent(str, location, tokens):
cur_col = col(location, str)
if cur_col > indent_stack[-1]:
indent_stack.append(cur_col)
else:
raise ParseException(str, location, "not a subentry")
def check_unindent(str, location, tokens):
if location >= len(str):
return
cur_col = col(location, str)
if not (cur_col < indent_stack[-1] and cur_col <= indent_stack[-2]):
raise ParseException(str, location, "not an unindent")
def do_unindent():
indent_stack.pop()
INDENT = lineEnd.suppress() + empty + empty.copy().setParseAction(
check_sub_indent)
UNDENT = FollowedBy(empty).setParseAction(check_unindent)
UNDENT.setParseAction(do_unindent)
json_id = (Word(alphas + "_", alphanums + "_") + Optional(oneOf("[0] [n]")))\
.setResultsName("json_id", listAllMatches=True)\
.setParseAction(lambda tokens: "".join(tokens))
aliases = delimitedList((Word(alphanums + "_") + Optional(oneOf("[0] [n]")))
.setParseAction(lambda tokens: "".join(tokens)))\
.setResultsName("aliases")
ident = Word(alphas + "_", alphanums + "_")
dict_def = originalTextFor(nestedExpr('{', '}'))
list_def = originalTextFor(nestedExpr('[', ']'))
dict_access = list_access = originalTextFor(ident + nestedExpr('[', ']'))
function_call = originalTextFor(
ZeroOrMore(ident + ".") + ident + nestedExpr('(', ')'))
python_allowed_expr = (dict_def ^ list_def ^ dict_access ^ \
list_access ^ function_call ^ restOfLine)\
.setResultsName("value", listAllMatches=True)
persistent_identifier = (Suppress("@persistent_identifier") + \
nestedExpr("(", ")"))\
.setResultsName("persistent_identifier")
legacy = (Suppress("@legacy") + originalTextFor(nestedExpr("(", ")")))\
.setResultsName("legacy", listAllMatches=True)
only_if = (Suppress("@only_if") + originalTextFor(nestedExpr("(", ")")))\
.setResultsName("only_if")
only_if_master_value = (Suppress("@only_if_value") + \
originalTextFor(nestedExpr("(", ")")))\
.setResultsName("only_if_master_value")
depends_on = (Suppress("@depends_on") + \
originalTextFor(nestedExpr("(", ")")))\
.setResultsName("depends_on")
parse_first = (Suppress("@parse_first") + \
originalTextFor(nestedExpr("(", ")")))\
.setResultsName("parse_first")
memoize = (Suppress("@memoize") + nestedExpr("(", ")"))\
.setResultsName("memoize")
field_decorator = parse_first ^ depends_on ^ only_if ^ \
only_if_master_value ^ memoize ^ legacy
#Independent decorators
inherit_from = (Suppress("@inherit_from") + \
originalTextFor(nestedExpr("(", ")")))\
.setResultsName("inherit_from")
override = (Suppress("@") + "override")\
.setResultsName("override")
extend = (Suppress("@") + "extend")\
.setResultsName("extend")
master_format = (Suppress("@master_format") + \
originalTextFor(nestedExpr("(", ")")))\
.setResultsName("master_format") \
.setParseAction(lambda toks: toks[0])
derived_calculated_body = (ZeroOrMore(field_decorator) + python_allowed_expr)\
.setResultsName('derived_calculated_def')
derived = "derived" + Suppress(":") + \
INDENT + derived_calculated_body + UNDENT
calculated = "calculated" + Suppress(":") + \
INDENT + derived_calculated_body + UNDENT
source_tag = quotedString\
.setParseAction(removeQuotes)\
.setResultsName("source_tag", listAllMatches=True)
source_format = Word(alphas, alphanums + "_")\
.setResultsName("source_format", listAllMatches=True)
creator_body = (ZeroOrMore(field_decorator) + source_format + \
Suppress(",") + source_tag + Suppress(",") + python_allowed_expr)\
.setResultsName("creator_def", listAllMatches=True)
creator = "creator" + Suppress(":") + \
INDENT + OneOrMore(creator_body) + UNDENT
field_def = (creator | derived | calculated)\
.setResultsName("type_field", listAllMatches=True)
#JsonExtra
json_dumps = (Suppress('dumps') + Suppress(',') + python_allowed_expr)\
.setResultsName("dumps")\
.setParseAction(lambda toks: toks.value[0])
json_loads = (Suppress("loads") + Suppress(",") + python_allowed_expr)\
.setResultsName("loads")\
.setParseAction(lambda toks: toks.value[0])
json_extra = (Suppress('json:') + \
INDENT + Each((json_dumps, json_loads)) + UNDENT)\
.setResultsName('json_ext')
#Checker
checker_function = (Optional(master_format) + ZeroOrMore(ident + ".") + ident + originalTextFor(nestedExpr('(', ')')))\
.setResultsName("checker", listAllMatches=True)
checker = ("checker" + Suppress(":") + INDENT +
OneOrMore(checker_function) + UNDENT)
#Description/Documentation
doc_double = QuotedString(quoteChar='"""', multiline=True)
doc_single = QuotedString(quoteChar="'''", multiline=True)
doc_string = INDENT + (doc_double | doc_single) + UNDENT
description_body = (Suppress('description:') + doc_string).\
setParseAction(lambda toks: toks[0][0])
description = (description_body | doc_double | doc_single)\
.setResultsName('description')
#Producer
producer_code = (Word(alphas, alphanums + "_")\
+ originalTextFor(nestedExpr("(", ")")))\
.setResultsName('producer_code', listAllMatches=True)
producer_body = (producer_code + Suppress(",") + python_allowed_expr)\
.setResultsName("producer_rule", listAllMatches=True)
producer = Suppress("producer:") + INDENT + OneOrMore(
producer_body) + UNDENT
schema = (Suppress('schema:') + INDENT + dict_def + UNDENT)\
.setParseAction(lambda toks: toks[0])\
.setResultsName('schema')
body = Optional(field_def) & Optional(checker) & Optional(json_extra) \
& Optional(description) & Optional(producer) & Optional(schema)
comment = Literal("#") + restOfLine + LineEnd()
include = (Suppress("include") + quotedString)\
.setResultsName("includes", listAllMatches=True)
rule = (Optional(persistent_identifier) + Optional(inherit_from) + \
Optional(override) + Optional(extend) +json_id + \
Optional(Suppress(",") + aliases) + Suppress(":") + \
INDENT + body + UNDENT)\
.setResultsName("rules", listAllMatches=True)
return OneOrMore(rule | include | comment.suppress())
