def genParseObject(self,cLocale):
sym = DateFormatSymbols(cLocale)
val = sym.getEras()
frag = Or(CaselessLiteral(val[0].encode('utf-8')))
for item in val[1:]:
frag.append(CaselessLiteral(item.encode('utf-8')))
return frag.setResultsName('era')
def _angular_distance():
_m = (usn + Literal("\'").leaveWhitespace())
_s = (usn + Literal("\"").leaveWhitespace())
ms = Or([_m + Optional(_s), _s])
ms = ms.setParseAction(lambda s, l, tok: AngularDistance(tok))
return ms
def create_post_timedelta_literal(tok):
"""Detects <plus/minus> <number> <timescale>"""
delta = Or(
[CaselessLiteral(t) for t in ['+', '-', 'plus', 'minus']]
) + originalTextFor(Or([
Word(nums) +
ZeroOrMore(',' + Word(nums+',')) +
ZeroOrMore('.' + Word(nums)),
CaselessLiteral('an'),
CaselessLiteral('a')
])) + CaselessLiteral(tok) + StringEnd()
delta.setName('post' + tok).\
setParseAction(DateParser.generate_post_timedelta)
return delta
def create_post_timedelta_literal(tok):
"""
Detects <plus/minus> <number> <timescale>
:param tok: the token we want to produce a detector for
:type tok: str
:return: the caseless literal
:rtype: pyparsing.Or
"""
delta = Or(
[CaselessLiteral(t) for t in ['+', '-', 'plus', 'minus']]
) + originalTextFor(Or([
Word(nums) +
ZeroOrMore(',' + Word(nums+',')) +
ZeroOrMore('.' + Word(nums)),
CaselessLiteral('an'),
CaselessLiteral('a')
])) + CaselessLiteral(tok) + StringEnd()
delta.setName('post' + tok).\
setParseAction(DateParser.generate_post_timedelta)
return delta
def get_precedence_list(self, parse_unary, parse_binary):
"""Build a precedence list to be used with pyparsing.infixNotation."""
precedence_list = []
for symbol_list in self.__ordered_operators:
expression = Or(symbol_list)
if type(self.__operators[symbol_list[0]]) is UnaryOperator:
arity = 1
parsing_function = parse_unary
associativity = opAssoc.RIGHT
else:
arity = 2
parsing_function = parse_binary
associativity = opAssoc.LEFT
entry = (expression, arity, associativity, parsing_function)
precedence_list.append(entry)
return precedence_list
def instance():
lit_e = CaselessLiteral('E')
plusorminus = Literal('+') | Literal('-')
number = Word(nums)
integer = Combine(Optional(plusorminus) +
number).setParseAction(lambda t: int(t[0]))
index = integer.copy().addParseAction(index_check(0))
floatnumber = Combine(integer + Optional(Literal('.') + Optional(number)) +
Optional(lit_e + integer)).setParseAction(
lambda t: float(t[0]))
#comment = Suppress("%") + Word(alphanums + " ")
comment = Regex(r"%.*").setName("comment").suppress()
linend = Or([comment, LineEnd()]).suppress()
section_end = (Literal('#') + LineEnd()).suppress()
vertex = (Group(
OneOrMore(floatnumber('point') + OneOrMore(White()).suppress())) +
linend)('vertex')
vertex_header = (Keyword('VERTEX') + linend).suppress()
vertex_section = (vertex_header + Group(OneOrMore(vertex))('vertices') +
section_end)
simplex = (
Group(OneOrMore(index('index') + OneOrMore(White()).suppress())) +
linend)('simplex')
simplex_header = (Keyword('SIMPLEX') + linend).suppress()
simplex_section = (simplex_header +
Group(OneOrMore(simplex))('simplices') + section_end)
boundarysegment = (Group(
index('id') +
OneOrMore(index('index') + OneOrMore(White()).suppress())) +
linend)('boundarysegment')
boundarysegment_header = (Keyword('BOUNDARYSEGMENTS') + linend).suppress()
boundarysegment_section = (
boundarysegment_header +
Dict(OneOrMore(boundarysegment))('boundarysegments') + section_end)
sections = Each([vertex_section, simplex_section, boundarysegment_section])
dgf_header = (Keyword('DGF') + linend).suppress()
dgf = (dgf_header + Dict(sections) + OneOrMore(section_end))('dgf')
return dgf
def __init__(self):
RegionPusher.__init__(self)
self.shape_definition = ds9_shape_defs
regionShape = define_shape_helper(self.shape_definition)
regionShape = regionShape.setParseAction(
lambda s, l, tok: Shape(tok[0], tok[1:]))
regionExpr = define_expr(
regionShape,
negate_func=lambda s, l, tok: tok[-1].set_exclude(),
)
coord_command_keys = "PHYSICAL IMAGE FK4 B1950 FK5 J2000 GALACTIC ECLIPTIC ICRS LINEAR AMPLIFIER DETECTOR".split(
)
coordCommand = define_simple_literals(
coord_command_keys,
parseAction=lambda s, l, tok: CoordCommand(tok[-1]))
regionGlobal = comment_shell_like(CaselessKeyword("global"),
lambda s, l, tok: Global(tok[-1]))
regionAtom = (regionExpr | coordCommand | regionGlobal)
regionAtom = regionAtom.setParseAction(self.pushAtom)
regionComment = comment_shell_like(Literal("#"),
parseAction=self.pushComment)
line_simple = define_line(atom=regionAtom,
separator=Literal(";"),
comment=regionComment)
line_w_composite = And([regionAtom,
CaselessKeyword("||").setParseAction(self.set_continued)
]) \
+ Optional(regionComment)
line = Or([line_simple, line_w_composite])
self.parser = Optional(line) + StringEnd()
def property_grammar():
ParserElement.setDefaultWhitespaceChars(' ')
model_props = [
Regex(r'LIBINPUT_MODEL_[_0-9A-Z]+')('NAME') - Suppress('=') -
(Literal('1'))('VALUE')
]
dimension = INTEGER('X') + Suppress('x') + INTEGER('Y')
sz_props = (
('LIBINPUT_ATTR_SIZE_HINT', Group(dimension('SETTINGS*'))),
('LIBINPUT_ATTR_RESOLUTION_HINT', Group(dimension('SETTINGS*'))),
)
size_props = [
Literal(name)('NAME') - Suppress('=') - val('VALUE')
for name, val in sz_props
]
grammar = Or(model_props + size_props)
return grammar
def make_one_header_assignment_parser(struct_parsers):
"""Create a parser that can parse a single header assignment
:Parameters:
- `struct_parsers`: a dictionary of parsers for structs
@return: a parser that can parse on header assignment
>>> test_parser = make_one_header_assignment_parser({})
>>>
>>> test_string = "a 42 32"
>>> parsed_assignments = test_parser.parseString(test_string)
>>> print parsed_assignments[0]['name']
a
>>> print parsed_assignments[0]['value']
42 32
"""
all_struct_parsers = Or([struct_parsers[s] for s in struct_parsers.keys()])
one_header_assignment_parser = NotAny(all_struct_parsers) + Group(
value_name('name') + restOfLine('value'))
one_header_assignment_parser.ignore(hash_comment)
return one_header_assignment_parser
def make_date_parser():
date_expr = ppc.iso8601_date.copy()
date_expr.setParseAction(ppc.convertToDate())
expr_last = (
CaselessKeyword('LAST') & ppc.integer.setResultsName('n')
& StringEnd()).setResultsName('interval').setParseAction(handle_last)
expr_prev = (CaselessKeyword('PREVIOUS') & Or(
CaselessKeyword('DAY').setResultsName('day')
| CaselessKeyword('WEEK').setResultsName('week')
| CaselessKeyword('MONTH').setResultsName('month')) + StringEnd()
).setResultsName('interval').setParseAction(handle_previous)
expr_fromto_date = (
CaselessKeyword('FROM') + date_expr.setResultsName('start') +
CaselessKeyword('TO') + date_expr.setResultsName('end') +
StringEnd()).setResultsName('interval').setParseAction(handle_fromto)
parser = expr_fromto_date | expr_last | expr_prev
return parser
def _make_mouse_parser():
from pyparsing import (Optional, Literal, Word, Group, Keyword, Or,
ZeroOrMore, Regex, Suppress)
double = Regex(r'-?\d+(\.\d*)?([eE]\d+)?')
coords = double + Suppress(Optional(Literal(','))) + double
integer = Word('0123456789')
move = ((Literal('(') + coords + Suppress(Literal(')'))) |
(Literal('[') + coords + Suppress(Literal(']'))) |
(Literal('<') + coords + Suppress(Literal('>'))))
buttons = ('left', 'middle', 'right', 'wheelup', 'wheeldown')
key = (Or([Keyword(sym) for sym in buttons]) | integer)
press = (
key +
Optional(Literal(':') +
(integer | (Literal('up') | Literal('down')))) +
Optional(Literal('/') + integer))
list_element = Group(move | press)
list_parser = list_element + ZeroOrMore(Suppress(',') + list_element)
return list_parser
def ruleParser():
global typedversion
# ------------------------------------------------------
# Atomic
# ------------------------------------------------------
# Tokens
lbrX = Literal("(")
rbrX = Literal(")")
commaX = Literal(",")
lbr = Literal("(").suppress()
rbr = Literal(")").suppress()
comma = Literal(",").suppress()
hash = Literal("#")
equ = Literal("=")
implies = Literal("=>").suppress()
dot = Literal(".").suppress()
eol = Literal("\n").suppress()
# Basic constructors
Alfabet = alphas + nums + "_$"
Number = Word(nums)
# Typeinfo/Constant
TypeInfo = oneOf("mr nonce pk sk fu table")
Constant = Word(alphas, Alfabet)
# Time
nTime = Number
xTime = Literal("xTime")
sTime = Literal("s") + lbrX + Number + rbrX
Time = Or([nTime, xTime, sTime])
# Const
Const = Forward()
ConstC = Literal("c") + lbrX + Constant + commaX + Time + rbrX
ConstF = Literal("c(ni,ni)")
Const << Or([Constant, ConstC, ConstF])
def stringize(s, l, t):
return ["".join(t)]
Const.setParseAction(stringize)
# Optional prime
def optprimeaction(s, l, t):
if len(t) == 0:
return [""]
else:
return t
optprime = Optional(Literal("'"))
optprime.setParseAction(optprimeaction)
# Two versions
if typedversion:
Variable = Word("x", Alfabet)
Variable = TypeInfo + lbr + Variable + rbr + optprime
Variable.setParseAction(lambda s, l, t: [If.Variable(t[0], t[1], t[2])])
else:
Variable = Word("x", Alfabet) + optprime
Variable.setParseAction(lambda s, l, t: [If.Variable("untyped", t[0], t[1])])
# Atomic
## DEVIANT : below there is an optprime after the atom. This
## is not in the BNF.
TypedConstant = TypeInfo + lbr + Const + rbr + optprime
TypedConstant.setParseAction(lambda s, l, t: [If.Constant(t[0], t[1], t[2])])
Atomic = Or(TypedConstant, Variable)
### TEST
# print Atomic.parseString("mr(Cas)'")
# print Atomic.parseString("nonce(Koen)")
# ------------------------------------------------------
# Messages
# ------------------------------------------------------
# Base forward declaration
Message = Forward()
# Agents etc
AgentMr = Literal("mr") + lbr + Const + rbr
AgentMr.setParseAction(lambda s, l, t: [If.Constant(t[0], t[1])])
Agent = Or([AgentMr, Variable])
# TODO Not implemented yet
KeyTable = Or([Literal("table") + lbr + Const + rbr, Variable])
KeyTableApp = Literal("tb") + lbr + KeyTable + comma + Agent + rbr + optprime
# Crypto
pkterm = Literal("pk") + lbr + Const + rbr + optprime
pkterm.setParseAction(lambda s, l, t: [If.PublicKey(t[0], t[1], t[2])])
##varterm = Variable + optprime ### Variable already has an optprime
varterm = Variable
Invertible = Or([pkterm, KeyTableApp, varterm])
PublicCypher = Literal("crypt") + lbr + Invertible + comma + Message + rbr
PublicCypher.setParseAction(lambda s, l, t: [If.PublicCrypt(t[1], t[2])])
XOR = Literal("rcrypt") + lbr + Message + comma + Message + rbr
XOR.setParseAction(lambda s, l, t: [If.XOR(t[1], t[2])])
SymmetricCypher = Literal("scrypt") + lbr + Message + comma + Message + rbr
SymmetricCypher.setParseAction(lambda s, l, t: [If.SymmetricCrypt(t[1], t[2])])
# TODO Not implemented yet
futerm = Or([Literal("fu") + lbr + Const + rbr, Variable])
Function = Literal("funct") + lbr + futerm + comma + Message + rbr
Concatenation = Literal("c").suppress() + lbr + Message + comma + Message + rbr
Concatenation.setParseAction(lambda s, l, t: [If.Composed(t[0], t[1])])
# Message composition
Composed = Or([Concatenation, SymmetricCypher, XOR, PublicCypher, Function, KeyTable, KeyTableApp])
Message << Or([Composed, Atomic])
### TEST
# print Message.parseString("nonce(c(Na,xTime))")
# ------------------------------------------------------
# Model of honest agents
# ------------------------------------------------------
Boolean = Or([Literal("true"), Literal("false"), Variable])
Session = Forward()
Session << Or([Literal("s") + lbr + Session + rbr, Number, Variable])
MsgEtc = Literal("etc")
MsgEtc.setParseAction(lambda s, l, t: [If.MsgList([If.Constant("special", "etc")])])
MsgVar = Group(Variable)
MsgVar.setParseAction(lambda s, l, t: [If.MsgList(t)])
MsgList = Forward()
MsgComp = Literal("c") + lbr + Message + comma + MsgList + rbr
MsgComp.setParseAction(lambda s, l, t: [If.MsgList([t[1]] + t[2].getList())])
MsgList << Or([MsgEtc, Variable, MsgComp])
Step = Or([Number, Variable])
### TEST
# print Message.parseString("xKb")
# print Message.parseString("mr(Cas)")
# print MsgList.parseString("etc")
# print MsgList.parseString("c(xKb,etc)")
# print MsgList.parseString("c(xA,c(xB,c(xKa,c(xKa',c(xKb,etc)))))")
# Principal fact
Principal = (
Literal("w")
+ lbr
+ Step
+ comma
+ Agent
+ comma
+ Agent
+ comma
+ MsgList
+ comma
+ MsgList
+ comma
+ Boolean
+ comma
+ Session
+ rbr
)
Principal.setParseAction(lambda s, l, t: [If.PrincipalFact(t[1:])])
# Message fact
MessageFact = (
Literal("m")
+ lbr
+ Step
+ comma
+ Agent
+ comma
+ Agent
+ comma
+ Agent
+ comma
+ Message
+ comma
+ Session
+ rbr
)
MessageFact.setParseAction(lambda s, l, t: [If.MessageFact(t[1:])])
# Goal fact
Secret = Literal("secret") + lbr + Message + Literal("f") + lbr + Session + rbr + rbr
Give = Literal("give") + lbr + Message + Literal("f") + lbr + Session + rbr + rbr
Witness = Literal("witness") + lbr + Agent + comma + Agent + comma + Constant + comma + Message + rbr
Request = Literal("request") + lbr + Agent + comma + Agent + comma + Constant + comma + Message + rbr
GoalFact = Or([Secret, Give, Witness, Request])
GoalFact.setParseAction(lambda s, l, t: [If.GoalFact(t)])
# Goal State
# It actually yields a rule (not a state per se)
Correspondence = Principal + dot + Principal
Correspondence.setParseAction(lambda s, l, t: [If.CorrespondenceRule(t)])
Secrecy = (
Literal("secret")
+ lbr
+ Literal("xsecret")
+ comma
+ Literal("f")
+ lbr
+ Session
+ rbr
+ rbr
+ dot
+ Literal("i")
+ lbr
+ Literal("xsecret")
+ rbr
)
Secrecy.setParseAction(lambda s, l, t: [If.SecrecyRule(t)])
STSecrecy = Literal("give(xsecret,f(xc)).secret(xsecret,f(xc))") + implies + Literal("i(xsecret)")
STSecrecy.setParseAction(lambda s, l, t: [If.STSecrecyRule(t)])
Authenticate = Literal("request") + lbr + Agent + comma + Agent + comma + Constant + comma + Message + rbr
Authenticate.setParseAction(lambda s, l, t: [If.AuthenticateRule(t)])
GoalState = Or([Correspondence, Secrecy, STSecrecy, Authenticate])
# TimeFact
TimeFact = Literal("h") + lbr + Message + rbr
TimeFact.setParseAction(lambda s, l, t: [If.TimeFact(t[1])])
# Intruder knowledge
IntruderKnowledge = Literal("i") + lbr + Message + rbr
# Facts and states
Fact = Or([Principal, MessageFact, IntruderKnowledge, TimeFact, Secret, Give, Witness, Request])
Fact.setParseAction(lambda s, l, t: [If.Fact(t)])
State = Group(delimitedList(Fact, ".")) ## From initial part of document, not in detailed BNF
State.setParseAction(lambda s, l, t: [If.State(t)])
# Rules
MFPrincipal = Or([MessageFact + dot + Principal, Principal])
mr1 = Literal("h") + lbr + Literal("s") + lbr + Literal("xTime") + rbr + rbr + dot + MFPrincipal
mr2 = implies
mr3 = Literal("h") + lbr + Literal("xTime") + rbr + dot + MFPrincipal + Optional(dot + delimitedList(GoalFact, "."))
MessageRule = Group(mr1) + mr2 + Group(mr3) ## DEVIANT : BNF requires newlines
MessageRule.setParseAction(lambda s, l, t: [If.MessageRule(t[0][3:], t[1][2:])])
InitialState = Literal("h") + lbr + Literal("xTime") + rbr + dot + State ## DEVIANT : BNF requires newlines
InitialState.setParseAction(lambda s, l, t: [If.InitialRule(t[2])])
# Intruder
IntruderRule = Literal("nogniet")
# Simplification
f_simplif = (
Literal("f")
+ lbr
+ Literal("s")
+ lbr
+ Literal("xc")
+ rbr
+ rbr
+ implies
+ Literal("f")
+ lbr
+ Literal("xc")
+ rbr
) ## DEVIANT : EOL removed
matching_request = Witness + dot + Request + implies
no_auth_intruder = Request + implies
SimplificationRule = Or([f_simplif, matching_request, no_auth_intruder])
# Compose all rules
Rule = Or([InitialState, MessageRule, IntruderRule, GoalState, SimplificationRule])
return Rule
def parse(cls,
content,
basedir=None,
resolve=True,
unresolved_value=DEFAULT_SUBSTITUTION):
"""parse a HOCON content
:param content: HOCON content to parse
:type content: basestring
:param resolve: if true, resolve substitutions
:type resolve: boolean
:param unresolved_value: assigned value to unresolved substitution.
If overriden with a default value, it will replace all unresolved values by the default value.
If it is set to pyhocon.STR_SUBSTITUTION then it will replace the value by its substitution expression (e.g., ${x})
:type unresolved_value: boolean
:return: a ConfigTree or a list
"""
unescape_pattern = re.compile(r'\\.')
def replace_escape_sequence(match):
value = match.group(0)
return cls.REPLACEMENTS.get(value, value)
def norm_string(value):
return unescape_pattern.sub(replace_escape_sequence, value)
def unescape_string(tokens):
return ConfigUnquotedString(norm_string(tokens[0]))
def parse_multi_string(tokens):
# remove the first and last 3 "
return tokens[0][3:-3]
def convert_number(tokens):
n = tokens[0]
try:
return int(n, 10)
except ValueError:
return float(n)
def convert_period(tokens):
period_value = int(tokens.value)
period_identifier = tokens.unit
period_unit = next((single_unit for single_unit, values in
cls.get_supported_period_type_map().items()
if period_identifier in values))
return period(period_value, period_unit)
# ${path} or ${?path} for optional substitution
SUBSTITUTION_PATTERN = r"\$\{(?P<optional>\?)?(?P<variable>[^}]+)\}(?P<ws>[ \t]*)"
def create_substitution(instring, loc, token):
# remove the ${ and }
match = re.match(SUBSTITUTION_PATTERN, token[0])
variable = match.group('variable')
ws = match.group('ws')
optional = match.group('optional') == '?'
substitution = ConfigSubstitution(variable, optional, ws, instring,
loc)
return substitution
# ${path} or ${?path} for optional substitution
STRING_PATTERN = '"(?P<value>(?:[^"\\\\]|\\\\.)*)"(?P<ws>[ \t]*)'
def create_quoted_string(instring, loc, token):
# remove the ${ and }
match = re.match(STRING_PATTERN, token[0])
value = norm_string(match.group('value'))
ws = match.group('ws')
return ConfigQuotedString(value, ws, instring, loc)
def include_config(instring, loc, token):
url = None
file = None
required = False
if token[0] == 'required':
required = True
final_tokens = token[1:]
else:
final_tokens = token
if len(final_tokens) == 1: # include "test"
value = final_tokens[0].value if isinstance(
final_tokens[0], ConfigQuotedString) else final_tokens[0]
if value.startswith("http://") or value.startswith(
"https://") or value.startswith("file://"):
url = value
else:
file = value
elif len(final_tokens) == 2: # include url("test") or file("test")
value = final_tokens[1].value if isinstance(
final_tokens[1], ConfigQuotedString) else final_tokens[1]
if final_tokens[0] == 'url':
url = value
elif final_tokens[0] == 'package':
file = cls.resolve_package_path(value)
else:
file = value
if url is not None:
logger.debug('Loading config from url %s', url)
obj = ConfigFactory.parse_URL(url,
resolve=False,
required=required,
unresolved_value=NO_SUBSTITUTION)
elif file is not None:
path = file if basedir is None else os.path.join(basedir, file)
def _make_prefix(path):
return ('<root>' if path is None else '[%s]' %
path).ljust(55).replace('\\', '/')
_prefix = _make_prefix(path)
def _load(path):
_prefix = _make_prefix(path)
logger.debug('%s Loading config from file %r', _prefix,
path)
obj = ConfigFactory.parse_file(
path,
resolve=False,
required=required,
unresolved_value=NO_SUBSTITUTION)
logger.debug('%s Result: %s', _prefix, obj)
return obj
if '*' in path or '?' in path:
paths = glob(path, recursive=True)
obj = None
def _merge(a, b):
if a is None or b is None:
return a or b
elif isinstance(a, ConfigTree) and isinstance(
b, ConfigTree):
return ConfigTree.merge_configs(a, b)
elif isinstance(a, list) and isinstance(b, list):
return a + b
else:
raise ConfigException(
'Unable to make such include (merging unexpected types: {a} and {b}',
a=type(a),
b=type(b))
logger.debug('%s Loading following configs: %s', _prefix,
paths)
for p in paths:
obj = _merge(obj, _load(p))
logger.debug('%s Result: %s', _prefix, obj)
else:
logger.debug('%s Loading single config: %s', _prefix, path)
obj = _load(path)
else:
raise ConfigException(
'No file or URL specified at: {loc}: {instring}',
loc=loc,
instring=instring)
return ConfigInclude(obj if isinstance(obj, list) else obj.items())
@contextlib.contextmanager
def set_default_white_spaces():
default = ParserElement.DEFAULT_WHITE_CHARS
ParserElement.setDefaultWhitespaceChars(' \t')
yield
ParserElement.setDefaultWhitespaceChars(default)
with set_default_white_spaces():
assign_expr = Forward()
true_expr = Keyword("true", caseless=True).setParseAction(
replaceWith(True))
false_expr = Keyword("false", caseless=True).setParseAction(
replaceWith(False))
null_expr = Keyword("null", caseless=True).setParseAction(
replaceWith(NoneValue()))
key = QuotedString(
'"', escChar='\\',
unquoteResults=False) | Word(alphanums + alphas8bit + '._- /')
eol = Word('\n\r').suppress()
eol_comma = Word('\n\r,').suppress()
comment = (Literal('#') | Literal('//')) - SkipTo(eol
| StringEnd())
comment_eol = Suppress(Optional(eol_comma) + comment)
comment_no_comma_eol = (comment | eol).suppress()
number_expr = Regex(
r'[+-]?(\d*\.\d+|\d+(\.\d+)?)([eE][+\-]?\d+)?(?=$|[ \t]*([\$\}\],#\n\r]|//))',
re.DOTALL).setParseAction(convert_number)
# Flatten the list of lists with unit strings.
period_types = list(
itertools.chain(*cls.get_supported_period_type_map().values()))
# `Or()` tries to match the longest expression if more expressions
# are matching. We employ this to match e.g.: 'weeks' so that we
# don't end up with 'w' and 'eeks'. Note that 'weeks' but also 'w'
# are valid unit identifiers.
# Allow only spaces as a valid separator between value and unit.
# E.g. \t as a separator is invalid: '10<TAB>weeks'.
period_expr = (
Word(nums)('value') + ZeroOrMore(White(ws=' ')).suppress() +
Or(period_types)('unit') +
WordEnd(alphanums).suppress()).setParseAction(convert_period)
# multi line string using """
# Using fix described in http://pyparsing.wikispaces.com/share/view/3778969
multiline_string = Regex(
'""".*?"*"""',
re.DOTALL | re.UNICODE).setParseAction(parse_multi_string)
# single quoted line string
quoted_string = Regex(
r'"(?:[^"\\\n]|\\.)*"[ \t]*',
re.UNICODE).setParseAction(create_quoted_string)
# unquoted string that takes the rest of the line until an optional comment
# we support .properties multiline support which is like this:
# line1 \
# line2 \
# so a backslash precedes the \n
unquoted_string = Regex(
r'(?:[^^`+?!@*&"\[\{\s\]\}#,=\$\\]|\\.)+[ \t]*',
re.UNICODE).setParseAction(unescape_string)
substitution_expr = Regex(r'[ \t]*\$\{[^\}]+\}[ \t]*'
).setParseAction(create_substitution)
string_expr = multiline_string | quoted_string | unquoted_string
value_expr = period_expr | number_expr | true_expr | false_expr | null_expr | string_expr
include_content = (quoted_string | (
(Keyword('url') | Keyword('file') | Keyword('package')) -
Literal('(').suppress() - quoted_string -
Literal(')').suppress()))
include_expr = (Keyword("include", caseless=True).suppress() +
(include_content |
(Keyword("required") - Literal('(').suppress() -
include_content - Literal(')').suppress()))
).setParseAction(include_config)
root_dict_expr = Forward()
dict_expr = Forward()
list_expr = Forward()
multi_value_expr = ZeroOrMore(comment_eol | include_expr
| substitution_expr | dict_expr
| list_expr | value_expr
| (Literal('\\') - eol).suppress())
# for a dictionary : or = is optional
# last zeroOrMore is because we can have t = {a:4} {b: 6} {c: 7} which is dictionary concatenation
inside_dict_expr = ConfigTreeParser(
ZeroOrMore(comment_eol | include_expr | assign_expr
| eol_comma))
inside_root_dict_expr = ConfigTreeParser(
ZeroOrMore(comment_eol | include_expr | assign_expr
| eol_comma),
root=True)
dict_expr << Suppress('{') - inside_dict_expr - Suppress('}')
root_dict_expr << Suppress('{') - inside_root_dict_expr - Suppress(
'}')
list_entry = ConcatenatedValueParser(multi_value_expr)
list_expr << Suppress('[') - ListParser(list_entry - ZeroOrMore(
eol_comma - list_entry)) - Suppress(']')
# special case when we have a value assignment where the string can potentially be the remainder of the line
assign_expr << Group(key - ZeroOrMore(comment_no_comma_eol) - (
dict_expr | (Literal('=') | Literal(':') | Literal('+=')) -
ZeroOrMore(comment_no_comma_eol) -
ConcatenatedValueParser(multi_value_expr)))
# the file can be { ... } where {} can be omitted or []
config_expr = ZeroOrMore(comment_eol | eol) + (
list_expr | root_dict_expr
| inside_root_dict_expr) + ZeroOrMore(comment_eol | eol_comma)
config = config_expr.parseString(content, parseAll=True)[0]
if resolve:
allow_unresolved = resolve and unresolved_value is not DEFAULT_SUBSTITUTION and unresolved_value is not MANDATORY_SUBSTITUTION
has_unresolved = cls.resolve_substitutions(
config, allow_unresolved)
if has_unresolved and unresolved_value is MANDATORY_SUBSTITUTION:
raise ConfigSubstitutionException(
'resolve cannot be set to True and unresolved_value to MANDATORY_SUBSTITUTION'
)
if unresolved_value is not NO_SUBSTITUTION and unresolved_value is not DEFAULT_SUBSTITUTION:
cls.unresolve_substitutions_to_value(config, unresolved_value)
return config
replaceWith,
upcaseTokens,
Word,
WordEnd,
WordStart,
ZeroOrMore,
)
from data_lists import tlds, schemes
alphanum_word_start = WordStart(wordChars=alphanums)
alphanum_word_end = WordEnd(wordChars=alphanums)
# the label definition ignores the fact that labels should not end in an hyphen
label = Word(initChars=alphanums, bodyChars=alphanums + '-', max=63)
domain_tld = Or(tlds)
domain_name = (alphanum_word_start + Combine(
Combine(OneOrMore(label + ('.' + FollowedBy(Word(alphanums + '-')))))
('domain_labels') + domain_tld('tld')) +
alphanum_word_end).setParseAction(downcaseTokens)
ipv4_section = (Word(
nums, asKeyword=True,
max=3).setParseAction(lambda x: str(int(x[0]))).addCondition(
lambda tokens: int(tokens[0]) < 256))
# basically, the grammar below says: start any words that start with a '.' or a number; I want to match words that start with a '.' because this will fail later in the grammar and I do not want to match anything that start with a '.'
ipv4_address = (alphanum_word_start + WordStart('.' + nums) +
Combine((ipv4_section + '.') * 3 + ipv4_section) +
NotAny(Regex('\.\S')) + alphanum_word_end)
hexadectet = Word(hexnums, min=1, max=4)
BOM = '\xef\xbb\xbf'
bom = Regex(BOM)
msg_utf8 = bom + utf_8_string
msg_any = utf_8_string
msg = Combine(Or([msg_utf8, msg_any])).setResultsName('MSG')
sd_name = CharsNotIn('= ]"', 1, 32)
param_name = sd_name.setResultsName('SD_PARAM_NAME')
param_value = QuotedString(quoteChar='"', escChar='\\', multiline=True)
param_value = param_value.setResultsName('SD_PARAM_VALUE')
sd_id = sd_name.setResultsName('SD_ID')
sd_param = Group(param_name + Regex('=') + param_value)
sd_params = Group(ZeroOrMore(Group(sp+sd_param.setResultsName('SD_PARAM'))))
sd_element = Group('['+sd_id+sd_params.setResultsName('SD_PARAMS')+']')
sd_element = sd_element.setResultsName('SD_ELEMENT')
sd_elements = Group(OneOrMore(sd_element))
structured_data = Or([nilvalue, sd_elements.setResultsName('SD_ELEMENTS')])
structured_data = structured_data.setResultsName('STRUCTURED_DATA')
time_hour = Regex('0[0-9]|1[0-9]|2[0-3]')
time_minute = Regex('[0-5][0-9]')
time_second = time_minute
time_secfrac = Regex('\.[0-9]{1,6}')
time_numoffset = Or([Regex('\+'), Regex('-')]) + \
time_hour + ':' + time_minute
time_offset = Or([Regex('Z'), time_numoffset])
partial_time = time_hour + ':' + time_minute + ':' + time_second + \
Optional(time_secfrac)
full_time = partial_time + time_offset
date_mday = Regex('[0-9]{2}')
date_month = Regex('0[1-9]|1[0-2]')
date_fullyear = Regex('[0-9]{4}')
full_date = date_fullyear + '-' + date_month + '-' + date_mday
See the LICENSE file in the root of this project for license details.
'''
from pyparsing import (
Word, alphas, alphanums, nums, cppStyleComment,
Keyword, Group, Optional, Or, OneOrMore, delimitedList, ZeroOrMore,
hexnums, Combine, FollowedBy, ParseException as pyparsingParseException
)
ParseException = pyparsingParseException
identifier = Word(alphas + "_", alphas + alphanums + "_")
qualified_identifier = Word(alphas + "_.", alphas + alphanums + "_.")
BASIC_TYPES = ["bool", "i8", "u8", "i16", "u16", "i32", "u32", "i64", "u64"]
basic_type = Or([Keyword(t) for t in BASIC_TYPES])
bit_width = Word(nums)
dec_literal = Word(nums)
hex_literal = Combine("0x" + Word(hexnums))
signed_literal = Combine(Optional('-') + (dec_literal ^ hex_literal))
comment = cppStyleComment
enumValue = Group(
Optional(comment)("doc") +
identifier("name") +
Optional('=' + signed_literal("constant"))
)
+ Suppress(Literal(":"))) \
+ Optional(name.setResultsName("interface") \
+ Suppress(Literal("."))) \
+ name_with_alias \
+ Optional(slice_enclosed \
| index_enclosed)).setResultsName("simple_mapping", listAllMatches=True)
## Broadcast of port mappings separeted by '|'
# and an optional '|' in the end (to support generated simple_mapping strings)
broadcast_mapping = simple_mapping \
+ ZeroOrMore(Suppress(Literal("|")) \
+ simple_mapping) + Optional("|")
## Index slice mapping of the form "index|slice=broadcast_mapping"
# Usually used for mapping signal indices and slices
index_slice_mapping = Group(Or([index.setResultsName("index"), slice]) \
+ Suppress(Literal("=")) \
+ broadcast_mapping \
).setResultsName("index_slice_mapping", listAllMatches=True)
## Mixed mapping of signal indices and slices separated by ';'
# and an optional ';' in the end (to support generated index_slice_mapping strings)
mixed_mapping = index_slice_mapping \
+ ZeroOrMore(Suppress(Literal(";")) \
+ index_slice_mapping) + Optional(";")
## Port mappings can be broadcast mapping in case of scalars and vectors
# and mixed mapping in case of vectors
port_mappings = mixed_mapping | broadcast_mapping
## Signal mapping of the form "signal=broadcast_maping|mixed_mapping"
from pyparsing import alphas, nums, restOfLine
from pyparsing import Forward, Keyword, QuotedString, Word
from pyparsing import OneOrMore, Optional, Or, ZeroOrMore
from pyparsing import ParserElement
from .data import Grammar, Expr, Token
from .encoders import languages
from .errors import InvalidQuantifier
from .errors import MultipleEvalCode
from .errors import MultipleStartSymbol, NoStartSymbol
from .errors import MultipleTypeDefinition
# Parser Elements -------------------------------------------------------------
_nsym = Word(alphas)
_tsym = '"' + Word(alphas) + '"'
_type = QuotedString('"')
_ltag = '@' + Or(languages)
_uops = '{' + Word(nums) + Optional(',' + Optional(Word(nums))) + '}'
_uops = _uops | '*' | '+' | '?'
_expr = Forward()
_elem = _nsym | _tsym | '(' + _expr + ')'
_fact = _elem + Optional(_uops)
_term = OneOrMore(_fact)
_expr << _term + ZeroOrMore('|' + _term)
_sdef = Keyword('START') + '=' + _nsym
_prod = _nsym + '->' + _expr
_tdef = _nsym + Optional(_ltag) + ':' + _type
_edef = Keyword('EVAL') + Optional(_ltag) + '=' + QuotedString('"', '\\')
_cmnt = ('#' + restOfLine).suppress()
_rule = (_sdef | _prod | _tdef | _edef) + ';' + Optional(_cmnt)
type_single = TYPE + EQUALITY + ASSET_NAME
type_list = generate_in_expr(TYPE, ASSET_NAME)
type_expr = type_list | type_single
cond_single = COND + EQUALITY + CONDITIONS
cond_list = generate_in_expr(COND, CONDITIONS)
cond_expr = cond_list | cond_single
path_single = PATH + EQUALITY + PATH_RE
path_list = generate_in_expr(PATH, PATH_RE)
path_expr = path_list | path_single
label_expr = LABELLED + EQUALITY + Or((TRUE, FALSE))
assy_expr = ASSY + EQUALITY + Or((TRUE, FALSE))
tristate_expr = Or(TRI_STATE_KEYS) + EQUALITY + Or((TRUE, FALSE, UNSET))
or_expr = Forward()
and_expr = Forward()
not_expr = Forward()
primary = Forward()
func_expr = Forward()
base_expr = (not_expr | code_expr | serial_expr | type_expr | cond_expr |
path_expr | label_expr | assy_expr | tristate_expr)
paren_expr = L_BRKT + or_expr + R_BRKT
primary << (base_expr | paren_expr)
func_expr << (FUNCTIONS + OF + func_expr | primary)
BOM = "\xef\xbb\xbf"
bom = Regex(BOM)
msg_utf8 = bom + utf_8_string
msg_any = utf_8_string
msg = Combine(Or([msg_utf8, msg_any])).setResultsName("MSG")
sd_name = CharsNotIn('= ]"', 1, 32)
param_name = sd_name.setResultsName("SD_PARAM_NAME")
param_value = QuotedString(quoteChar='"', escChar="\\", multiline=True)
param_value = param_value.setResultsName("SD_PARAM_VALUE")
sd_id = sd_name.setResultsName("SD_ID")
sd_param = Group(param_name + Regex("=") + param_value)
sd_params = Group(ZeroOrMore(Group(sp + sd_param.setResultsName("SD_PARAM"))))
sd_element = Group("[" + sd_id + sd_params.setResultsName("SD_PARAMS") + "]")
sd_element = sd_element.setResultsName("SD_ELEMENT")
sd_elements = Group(OneOrMore(sd_element))
structured_data = Or([nilvalue, sd_elements.setResultsName("SD_ELEMENTS")])
structured_data = structured_data.setResultsName("STRUCTURED_DATA")
time_hour = Regex("0[0-9]|1[0-9]|2[0-3]")
time_minute = Regex("[0-5][0-9]")
time_second = time_minute
time_secfrac = Regex("\.[0-9]{1,6}")
time_numoffset = Or([Regex("\+"), Regex("-")]) + time_hour + ":" + time_minute
time_offset = Or([Regex("Z"), time_numoffset])
partial_time = time_hour + ":" + time_minute + ":" + time_second + Optional(time_secfrac)
full_time = partial_time + time_offset
date_mday = Regex("[0-9]{2}")
date_month = Regex("0[1-9]|1[0-2]")
date_fullyear = Regex("[0-9]{4}")
full_date = date_fullyear + "-" + date_month + "-" + date_mday
timestamp = Combine(Or([nilvalue, full_date + "T" + full_time]))
timestamp = timestamp.setResultsName("TIMESTAMP")
attrexpr << varexpr + Literal(".") + name
valexpr << Group(QuotedString("\"") | nums | attrexpr)
"""
provides_decl = Keyword("provides") - Suppress(open_block) - Group(ZeroOrMore(Word(printables, excludeChars="{}"))) - Suppress(close_block)
requires_decl = Keyword("requires") - Suppress(open_block) - Group(ZeroOrMore(Word(printables, excludeChars="{}"))) - Suppress(close_block)
agent_decl = Keyword("agent") + Suppress(open_block) + Group(ZeroOrMore(Word(printables, excludeChars="{}"))) + Suppress(close_block)
script_decl = Keyword("script") + Suppress(open_block) + QuotedString("\"", multiline=True) + Suppress(close_block)
tool_decl = Keyword("tool") + Suppress(open_block) + QuotedString("\"", multiline=True) + Suppress(close_block)
action_contents = Group(provides_decl ^ requires_decl ^ agent_decl ^ script_decl ^ tool_decl)
action_decl = Group(Keyword("action") + name.setResultsName("actionName") + Optional(action_keywords) + Suppress(open_block) - ZeroOrMore(action_contents) - Suppress(close_block)).setParseAction(add_action)
primary_decl = Forward()
block = Forward()
primary_keyword = Or([Keyword("branch"), Keyword("iteration"), Keyword("selection"), Keyword("sequence"), Keyword("task")])
primary_decl << Group(primary_keyword.setParseAction(add_primary) - Optional(name)("name") - Suppress(open_block).setParseAction(start_prim) - ZeroOrMore( (action_decl ^ primary_decl) ) - Suppress(close_block).setParseAction(stop_prim))
process_decl = (Keyword("process") + name.setResultsName("processName").setParseAction(process_name) + Suppress(open_block) - ZeroOrMore( (primary_decl ^ action_decl )).setResultsName("block") - Suppress(close_block))
def arr_to_json(orig_arr):
arr = copy.deepcopy(orig_arr)
global glob_cur_path
global glob_d
d = copy.deepcopy(glob_d)
cur_path = d["process"]["contains"]
glob_d = {"process": {"contains": {}, "type": "process", "name": ""}}
glob_cur_path = glob_d["process"]["contains"]
stack = []
count = -1
EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
author = """Vincent Kraeutler, 2007"""
from netaddress.rfc3986 import port, host, path_abempty
from pyparsing import Combine, Literal, Optional, Or
scheme = Or([Literal("http"), Literal("https")]).setResultsName("scheme")
prefix = scheme + Literal("://")
path = path_abempty.setResultsName("path")
angelPort = Optional(Literal(":") + port)
angelPath = Optional(Combine(path), default = "/")
angelHost = Optional(Combine(host), default = "localhost").setResultsName("host")
# URI grammar as used in angel-app
angelURI = prefix + angelHost + angelPort + angelPath
def parse(uri = ""):
#try:
return angelURI.parseString(uri)
#except:
# raise ValueError, "Failed to parse URI: " + `uri`
name = (LineStart() + Literal("Name:")).suppress()
pre = Literal("Pre:").suppress()
num = Combine(Optional(Literal("-")) + Word(nums))
assign = Literal("=").suppress()
constant = Combine(Literal("C") + Optional(Word(nums)))
skip = Literal("skip")
to = Literal("to").suppress()
percent = Literal("%").suppress()
ident = Combine(percent + Word(alphanums))
funName = Word(srange("[a-zA-Z]"), alphanums)
lpar = Literal( "(" ).suppress()
rpar = Literal( ")" ).suppress()
comma = Literal( "," ).suppress()
# theory expression
unops = ["~", "-"]
lBvUnop = Or(mkLit(s) for s in unops)
unops = ["!"]
lUnop = Or(mkLit(s) for s in unops)
binops = ["&", "|", "^", ">>", "u>>", "<<", "+", "-", "*", "/", "/u", "%", "%u"]
lBvBinop = Or(mkLit(s) for s in binops)
binops = ["&&", "||"]
lBinop = Or(mkLit(s) for s in binops)
cmps = ["==", "!=", "<", ">", "u<", "u>", "<=", ">=", "u<=", "u>="]
lCmp = Or(mkLit(s) for s in cmps)
funs = ["abs", "computeKnownZeroBits", "countLeadingZeros", \
"countTrailingZeros", "computeNumSignBits", "log2", "lshr", \
"max", "umax", "width"]
xfuns = ["sext", "trunc", "zext", "ZExtOrTrunc"]
lBvFun = Or(mkLit(s) for s in funs + xfuns)
xFun = Or(mkLit(s) for s in xfuns)
funs = ["equivalentAddressValues", "hasOneUse", "isPowerOf2", \
def property_grammar():
ParserElement.setDefaultWhitespaceChars(' ')
dpi_setting = Group(
Optional('*')('DEFAULT') + INTEGER('DPI') + Suppress('@') +
INTEGER('HZ'))('SETTINGS*')
mount_matrix_row = SIGNED_REAL + ',' + SIGNED_REAL + ',' + SIGNED_REAL
mount_matrix = Group(mount_matrix_row + ';' + mount_matrix_row + ';' +
mount_matrix_row)('MOUNT_MATRIX')
xkb_setting = Optional(Word(alphanums + '+-/@._'))
# Although this set doesn't cover all of characters in database entries, it's enough for test targets.
name_literal = Word(printables + ' ')
props = (
('MOUSE_DPI', Group(OneOrMore(dpi_setting))),
('MOUSE_WHEEL_CLICK_ANGLE', INTEGER),
('MOUSE_WHEEL_CLICK_ANGLE_HORIZONTAL', INTEGER),
('MOUSE_WHEEL_CLICK_COUNT', INTEGER),
('MOUSE_WHEEL_CLICK_COUNT_HORIZONTAL', INTEGER),
('ID_AUTOSUSPEND', Or((Literal('0'), Literal('1')))),
('ID_INPUT', Or((Literal('0'), Literal('1')))),
('ID_INPUT_ACCELEROMETER', Or((Literal('0'), Literal('1')))),
('ID_INPUT_JOYSTICK', Or((Literal('0'), Literal('1')))),
('ID_INPUT_KEY', Or((Literal('0'), Literal('1')))),
('ID_INPUT_KEYBOARD', Or((Literal('0'), Literal('1')))),
('ID_INPUT_MOUSE', Or((Literal('0'), Literal('1')))),
('ID_INPUT_POINTINGSTICK', Or((Literal('0'), Literal('1')))),
('ID_INPUT_SWITCH', Or((Literal('0'), Literal('1')))),
('ID_INPUT_TABLET', Or((Literal('0'), Literal('1')))),
('ID_INPUT_TABLET_PAD', Or((Literal('0'), Literal('1')))),
('ID_INPUT_TOUCHPAD', Or((Literal('0'), Literal('1')))),
('ID_INPUT_TOUCHSCREEN', Or((Literal('0'), Literal('1')))),
('ID_INPUT_TRACKBALL', Or((Literal('0'), Literal('1')))),
('POINTINGSTICK_SENSITIVITY', INTEGER),
('POINTINGSTICK_CONST_ACCEL', REAL),
('ID_INPUT_JOYSTICK_INTEGRATION', Or(('internal', 'external'))),
('ID_INPUT_TOUCHPAD_INTEGRATION', Or(('internal', 'external'))),
('XKB_FIXED_LAYOUT', xkb_setting),
('XKB_FIXED_VARIANT', xkb_setting),
('XKB_FIXED_MODEL', xkb_setting),
('KEYBOARD_LED_NUMLOCK', Literal('0')),
('KEYBOARD_LED_CAPSLOCK', Literal('0')),
('ACCEL_MOUNT_MATRIX', mount_matrix),
('ACCEL_LOCATION', Or(('display', 'base'))),
('PROXIMITY_NEAR_LEVEL', INTEGER),
('IEEE1394_UNIT_FUNCTION_MIDI', Or((Literal('0'), Literal('1')))),
('IEEE1394_UNIT_FUNCTION_AUDIO', Or((Literal('0'), Literal('1')))),
('IEEE1394_UNIT_FUNCTION_VIDEO', Or((Literal('0'), Literal('1')))),
('ID_VENDOR_FROM_DATABASE', name_literal),
('ID_MODEL_FROM_DATABASE', name_literal),
)
fixed_props = [
Literal(name)('NAME') - Suppress('=') - val('VALUE')
for name, val in props
]
kbd_props = [
Regex(r'KEYBOARD_KEY_[0-9a-f]+')('NAME') - Suppress('=') -
('!' ^ (Optional('!') - Word(alphanums + '_')))('VALUE')
]
abs_props = [
Regex(r'EVDEV_ABS_[0-9a-f]{2}')('NAME') - Suppress('=') -
Word(nums + ':')('VALUE')
]
grammar = Or(fixed_props + kbd_props + abs_props) + EOL
return grammar
def _build_filter_parser():
field = _build_field_expr()
negation = CaselessKeyword('not')
negation.setParseAction(lambda x: Negation(x[0]))
comparison_operator = Group(
Keyword('=') ^ Keyword('!=') ^ Keyword('>=') ^ Keyword('<=')
^ Keyword('<') ^ Keyword('>') ^ Keyword('lte') # match before lt
^ Keyword('lt') ^ Keyword('gte') # match before gt
^ Keyword('gt')
^ (Optional(negation) +
(CaselessKeyword('contains') ^ CaselessKeyword('icontains')
^ CaselessKeyword('startswith') ^ CaselessKeyword('istartswith')
^ CaselessKeyword('endswith') ^ CaselessKeyword('iendswith')
^ CaselessKeyword('eq'))))
comparison_operator.setParseAction(lambda x: Operator(x))
single_value_operator = Group(
CaselessKeyword('isnull')
^ (Optional(negation) + CaselessKeyword('isnull')))
single_value_operator.setParseAction(lambda x: Operator(x))
plusorminus = Literal('+') | Literal('-')
num_integer = Combine(Optional(plusorminus) + Word(nums))
num_integer.setParseAction(lambda x: Integer(x[0]))
num_float = Combine(Optional(plusorminus) + Word(nums) + '.' + Word(nums))
num_float.setParseAction(lambda x: Float(x[0]))
quoted_string = (QuotedString("'") ^ QuotedString('"'))
quoted_string.setParseAction(lambda x: String(x[0]))
boolean = Or([
CaselessKeyword(v) for v in BOOLEAN_TRUE_VALUES + BOOLEAN_FALSE_VALUES
])
boolean.setParseAction(lambda x: Boolean(x[0]))
value = (quoted_string ^ num_integer ^ num_float ^ boolean)
comparison = Group((field + comparison_operator + value)
^ (value + comparison_operator + field)
^ (field + comparison_operator + field)
^ (field + single_value_operator))
comparison.setParseAction(lambda x: Comparison(x))
invalid_comparison = Group(
(value + comparison_operator + value).setParseAction(lambda x: fail(
"Value may not be compared with values: {0}".format(' '.join(x)))))
logical_op = Group(CaselessKeyword("and") | CaselessKeyword("or"))
logical_op.setParseAction(lambda x: LogicalOp(x[0][0]))
expr = Forward()
statement = (comparison
| invalid_comparison
| (Literal('(').suppress() + expr + Literal(')').suppress()))
statement.setParseAction(lambda x: Statement(x))
expr << statement + ZeroOrMore(logical_op + statement)
return expr
def np_uint(s, loc, tokens):
return np_composite('uint', np_uint_dtypes)
def np_int(s, loc, tokens):
return np_composite('int', np_int_dtypes)
def np_float(s, loc, tokens):
return np_composite('float', np_float_dtypes)
dtype_checks.append(Keyword('int').setParseAction(np_int))
dtype_checks.append(Keyword('uint').setParseAction(np_uint))
dtype_checks.append(Keyword('float').setParseAction(np_float))
composite = ['int', 'uint', 'float']
ndarray_simple_contract = Or(dtype_checks + array_constraints)
ndarray_simple_contract.setName('numpy element contract')
suggester = create_suggester(get_options=lambda: atomic + composite)
baseExpr = ndarray_simple_contract | suggester
baseExpr.setName('numpy contract (with recovery)')
op = myOperatorPrecedence
# op = operatorPrecedence
ndarray_composite_contract = op(baseExpr, [
(',', 2, opAssoc.LEFT, ArrayAnd.parse_action),
('|', 2, opAssoc.LEFT, ArrayOR.parse_action),
])
def my_delim_list2(what, delim):
minus = Literal("-")
comma = Literal(",")
lbracket = Literal("[")
rbracket = Literal("]")
lbrace = Literal("{")
rbrace = Literal("}")
hashsign = Literal("#")
exclamation = Literal("!")
caret = Literal("^")
hex_num = Combine(Literal("0x") + Word("0123456789abcdef"))
dec_num = Word("0123456789")
# Operand Parsing
# ============================================================================ #
sign = Optional(Or([plus, minus("minus")]))
immediate = Group(Optional(Suppress(hashsign)) + (sign + Or([hex_num, dec_num]))("value"))
register = Group(Or([
Combine(Literal("r") + Word(nums)("reg_num")),
Combine(Literal("d") + Word(nums)("reg_num")),
Combine(Literal("c") + Word(nums)("reg_num")),
Combine(Literal("p") + Word(nums)("reg_num")),
Literal("sp"),
Literal("lr"),
Literal("pc"),
Literal("fp"),
Literal("ip"),
Literal("sl"),
Literal("sb"),
this <<= operator_expression | last
last = this
# Set the full expression and return it
expression <<= last
return expression
# An integer value
integer = Word(nums)
integer.setParseAction(Integer.parse)
integer.setName("integer")
dice_separators = RandomElement.DICE_MAP.keys()
dice_element = Or(
wrap_string(CaselessLiteral, x, suppress=False) for x in dice_separators
)
special = wrap_string(Literal, "%", suppress=False) | wrap_string(
CaselessLiteral, "f", suppress=False
)
# An expression in dice notation
expression = (
StringStart()
+ operatorPrecedence(
integer,
[
(dice_element, 2, opAssoc.LEFT, RandomElement.parse, special),
(dice_element, 1, opAssoc.RIGHT, RandomElement.parse_unary, special),
(wrap_string(CaselessLiteral, "x"), 2, opAssoc.LEFT, Explode.parse),
(wrap_string(CaselessLiteral, "x"), 1, opAssoc.LEFT, Explode.parse),
from pyparsing import CaselessLiteral, Word, Optional, Group, nums, alphanums, oneOf, \
ZeroOrMore, StringEnd, Or, Suppress, Forward, ParseException as pyPE
from dbp.db import Update, Insert, Delete
from dbp.where import _combinators, _ops, ParseException
# Example syntax
# (SELECT, UPDATE) (values) WHERE (column operator value)
# INSERT (values)
# DELETE WHERE ...
# INSERT (values) SELECT ? probs not.
_operators = ("SELECT", "UPDATE")
operators = Or(CaselessLiteral(o) for o in _operators)
combinators = Or(CaselessLiteral(c) for c in _combinators)
# comma separated values
w = Word(alphanums)
c = Suppress(",")
values = w + ZeroOrMore(c + w)
# Grammar:
# B -> AND | OR
# T -> f op f | ( C )
# C -> T B T | T
# S -> WHERE C EOF
stringLit = Group('"' + w + '"')
intLit = Word(nums)
C = Forward()
T = ((stringLit | intLit | w) + oneOf(_ops) +
(stringLit | intLit | w)) | (Suppress("(") + C + Suppress(")"))
expression = operatorPrecedence(term,
[(oneOf("* /"), 2, opAssoc.LEFT, parse_expression),
(oneOf("+ -"), 2, opAssoc.LEFT, parse_expression)]
)
assignment_stmt = variable + equals + expression + StringEnd()
assignment_stmt.setParseAction(parse_assignment_stmt)
output_stmt = output_fn + expression + StringEnd()
output_stmt.setParseAction(parse_output_stmt)
input_stmt = input_fn + term + StringEnd()
input_stmt.setParseAction(parse_input_stmt)
stmt = Or([assignment_stmt, input_stmt, output_stmt])
stmt.setParseAction(print_tree)
def parser_init():
global symbol_table
symbol_table = {}
def test_negative():
parser_init()
test_negative_string("x=1 +")
test_negative_string("x==1 ** x /")
test_negative_string("z 1 3 + 4")
test_negative_string("z =1 3 + 4")
test_negative_string("z =1 3 + 4;")
def test_negative_string(expr):
lbracket = Literal("[")
rbracket = Literal("]")
colon = Literal(":")
hex_num = Combine(Literal("0x") + Word("0123456789abcdef"))
dec_num = Word("0123456789")
# Operand Parsing
# ============================================================================ #
modifier = (Optional(
Or([
Literal("ymmword"),
Literal("xmmword"),
Literal("xword"),
Literal("tword"),
Literal("qword"),
Literal("dword"),
Literal("word"),
Literal("byte"),
Literal("far"),
])) + Optional(Literal("ptr")))
immediate = Optional("-") + Or([hex_num, dec_num])
segment = Or([
Literal("cs"),
Literal("ds"),
Literal("ss"),
Literal("es"),
Literal("fs"),
Literal("gs"),
bullet = originalTextFor(number + Literal('.'))
percent = originalTextFor(number + Literal('%'))
hypen_word = Combine(word + Literal('-') + word)
apos_word1 = Combine(Literal('\'') + word)
apos_word2 = Combine(word + Literal('\'') + word)
new_line = Literal('\n')
tab = Literal('\t')
punctuation = list(u"‘’.,;„“”()‘:\"\'`′!?-–—…") + ['...']
punkt = map(Literal, punctuation)
all_ = [word, initial, bullet, hypen_word, apos_word1, apos_word2, \
number, percent, new_line, latin_word, tab]
all_.extend(punkt)
all_ = Or(all_).parseWithTabs()
def tokenize(feed, include_junk = True, echo_junk = False):
"""Returns list of tokens"""
tokens = []
start = 0
for t, s, e in all_.scanString(feed):
if s != 0:
junk = feed[start:s].strip()
if junk != "":
if include_junk: tokens.append(junk)
if echo_junk: print(junk)
if t[0]:
tokens.append(t[0])
start = e
def np_int(s, loc, tokens):
return np_composite('int', np_int_dtypes)
def np_float(s, loc, tokens):
return np_composite('float', np_float_dtypes)
dtype_checks.append(Keyword('int').setParseAction(np_int))
dtype_checks.append(Keyword('uint').setParseAction(np_uint))
dtype_checks.append(Keyword('float').setParseAction(np_float))
composite = ['int', 'uint', 'float']
ndarray_simple_contract = Or(dtype_checks + array_constraints)
ndarray_simple_contract.setName('numpy element contract')
suggester = create_suggester(get_options=lambda: atomic + composite)
baseExpr = ndarray_simple_contract | suggester
baseExpr.setName('numpy contract (with recovery)')
op = myOperatorPrecedence
# op = operatorPrecedence
ndarray_composite_contract = op(baseExpr, [
(',', 2, opAssoc.LEFT, ArrayAnd.parse_action),
('|', 2, opAssoc.LEFT, ArrayOR.parse_action),
])
def my_delim_list2(what, delim):
def someOf(*argv):
return OneOrMore(Or(argv))
from pdfile import pdfile
import sys
"""
A compiler from message definitions into Pd patches.
"""
# reserved words
reserved_words = ['name', 'slots', 'constants', 'equations', 'output']
reserved_parsers = []
for reserved in reserved_words:
literal = Literal(reserved.upper())
globals()[reserved] = literal
reserved_parsers.append(literal)
reserved = Or(reserved_parsers)
word = ~reserved + Word(alphanums + "_" + "-")
function = Literal("lexicon")
equation = word + "=" + function + "(" + OneOrMore(word) + \
Optional(Literal(";") + OneOrMore(word + Optional(Literal("'")))) + ")"
def parse_equation(toks):
name = toks[0]
function = toks[2]
first_slot = []
second_slot = []
in_first = True
position = 4
while position < len(toks) - 1:
def ruleParser():
global typedversion
# ------------------------------------------------------
# Atomic
# ------------------------------------------------------
# Tokens
lbr = Literal("(")
rbr = Literal(")")
comma = Literal(",")
hash = Literal("#")
equ = Literal("=")
implies = Literal("=>")
dot = Literal(".")
eol = Literal("\n").suppress()
# Basic constructors
Alfabet = alphas + nums + "_$"
Number = Word(nums)
Number.setParseAction(lambda s, l, t: ["number", Term.TermConstant(t[0])])
# Typeinfo/Constant
TypeInfo = oneOf("mr nonce pk sk fu table")
TypeInfo.setParseAction(
lambda s, l, t: ["typeinfo", Term.TermConstant(t[0])])
Constant = Word(alphas, Alfabet)
Constant.setParseAction(lambda s, l, t: [Term.TermConstant(t[0])])
# Time
nTime = Number
xTime = Literal("xTime")
sTime = Literal("s").suppress() + lbr + Group(Number) + rbr
Time = Or([nTime, xTime, sTime])
# Const
Const = Forward()
ConstC = Literal("c") + lbr + Constant + comma + Time + rbr
ConstF = Literal("c(ni,ni)")
Const << Or([Constant, ConstC, ConstF])
Const.setParseAction(lambda s, l, t: [If.Constant("".join(t))])
# Two versions
Variable = Word("x", Alfabet)
Variable.setParseAction(
lambda s, l, t: [Term.TermVariable(t[0] + "V", None)])
if typedversion:
Variable = TypeInfo + lbr + Variable + rbr
# Optional prime
optprime = Optional(Literal("'"))
# Atomic
## DEVIANT : below there is an optprime after the atom. This
## is not in the BNF.
Atomic = Or([TypeInfo + lbr + Const + rbr, Variable]) + optprime
### TEST
#print Time.parseString("s(25)")
#print Variable.parseString("xCas")
#print Atomic.parseString("nonce(Koen)")
# ------------------------------------------------------
# Messages
# ------------------------------------------------------
# Base forward declaration
Message = Forward()
# Agents etc
Agent = Or([Literal("mr") + lbr + Const + rbr, Variable])
KeyTable = Or([Literal("table") + lbr + Const + rbr, Variable])
KeyTableApp = Literal(
"tb") + lbr + KeyTable + comma + Agent + rbr + optprime
# Crypto
pkterm = Literal("pk") + lbr + Const + rbr + optprime
varterm = Variable + optprime
Invertible = Or([pkterm, KeyTableApp, varterm])
PublicCypher = Literal("crypt") + lbr + Invertible + comma + Message + rbr
PublicCypher.setParseAction(lambda s, l, t: [Term.TermEncrypt(t[2], t[1])])
XOR = Literal("rcrypt") + lbr + Message + comma + Message + rbr
SymmetricCypher = Literal("scrypt") + lbr + Message + comma + Message + rbr
futerm = Or([Literal("fu") + lbr + Const + rbr, Variable])
Function = Literal("funct") + lbr + futerm + comma + Message + rbr
# Message composition
Concatenation = Literal("c") + lbr + Message + comma + Message + rbr
Concatenation.setParseAction(lambda s, l, t: [Term.TermTuple(t[1], t[2])])
Composed = Or([
Concatenation, SymmetricCypher, XOR, PublicCypher, Function, KeyTable,
KeyTableApp
])
Message << Or([Composed, Atomic])
### TEST
#print Message.parseString("nonce(c(Na,xTime))")
# ------------------------------------------------------
# Model of honest agents
# ------------------------------------------------------
Boolean = Or([Literal("true"), Literal("false"), Variable])
Session = Forward()
Session << Or([Literal("s") + lbr + Session + rbr, Number, Variable])
MsgEtc = Literal("etc")
MsgList = Forward()
MsgComp = Literal("c") + lbr + Message + comma + MsgList + rbr
MsgList << Or([MsgEtc, Variable, MsgComp])
Step = Or([Number, Variable])
### TEST
#print Message.parseString("xKb")
#print MsgList.parseString("etc")
#print MsgList.parseString("c(xKb,etc)")
#print MsgList.parseString("c(xA,c(xB,c(xKa,c(xKa',c(xKb,etc)))))")
# Principal fact
Principal = Literal(
"w"
) + lbr + Step + comma + Agent + comma + Agent + comma + MsgList + comma + MsgList + comma + Boolean + comma + Session + rbr
Principal.setParseAction(lambda s, l, t: ["Principal", t])
# Message fact
MessageFact = Literal(
"m"
) + lbr + Step + comma + Agent + comma + Agent + comma + Agent + comma + Message + comma + Session + rbr
# Goal fact
Correspondence = Principal + dot + Principal
Secret = Literal("secret") + lbr + Message + Literal(
"f") + lbr + Session + rbr + rbr
Secrecy = Literal("secret") + lbr + Literal("xsecret") + comma + Literal(
"f") + lbr + Session + rbr + rbr + dot + Literal("i") + lbr + Literal(
"xsecret") + rbr
Give = Literal("give") + lbr + Message + Literal(
"f") + lbr + Session + rbr + rbr
STSecrecy = Literal("give(xsecret,f(xc)).secret(xsecret,f(xc))"
) + implies + Literal("i(xsecret)")
Witness = Literal(
"witness"
) + lbr + Agent + comma + Agent + comma + Constant + comma + Message + rbr
Request = Literal(
"request"
) + lbr + Agent + comma + Agent + comma + Constant + comma + Message + rbr
Authenticate = Literal(
"request"
) + lbr + Agent + comma + Agent + comma + Constant + comma + Message + rbr
GoalState = Or([Correspondence, Secrecy, STSecrecy, Authenticate])
GoalFact = Or([Secret, Give, Witness, Request])
# TimeFact
TimeFact = Literal("h") + lbr + Message + rbr
# Intruder knowledge
IntruderKnowledge = Literal("i") + lbr + Message + rbr
# Facts and states
Fact = Or([
Principal, MessageFact, IntruderKnowledge, TimeFact, Secret, Give,
Witness, Request
])
State = Group(delimitedList(
Fact, ".")) ## From initial part of document, not in detailed BNF
# Rules
MFPrincipal = Or([MessageFact + dot + Principal, Principal])
mr1 = Literal("h") + lbr + Literal("s") + lbr + Literal(
"xTime") + rbr + rbr + dot + MFPrincipal
mr2 = implies
mr3 = Literal("h") + lbr + Literal(
"xTime") + rbr + dot + MFPrincipal + Optional(
dot + delimitedList(GoalFact, "."))
MessageRule = Group(mr1) + mr2 + Group(
mr3) ## DEVIANT : BNF requires newlines
InitialState = Literal("h") + lbr + Literal(
"xTime") + rbr + dot + State ## DEVIANT : BNF requires newlines
# Intruder
IntruderRule = Literal("nogniet")
# Simplification
f_simplif = Literal("f") + lbr + Literal("s") + lbr + Literal(
"xc") + rbr + rbr + implies + Literal("f") + lbr + Literal(
"xc") + rbr ## DEVIANT : EOL removed
matching_request = Witness + dot + Request + implies
no_auth_intruder = Request + implies
SimplificationRule = Or([f_simplif, matching_request, no_auth_intruder])
# Compose all rules
Rule = Or([
InitialState, MessageRule, IntruderRule, GoalState, SimplificationRule
])
return Rule
literal_kw = true_kw | false_kw
# RESERVED
reserved = statement_keyword | marker_keyword | operator_keyword | literal_kw
# IDENTIFIER
identifier_regex = r"(?:[a-zA-Z]|_[a-zA-Z])[a-zA-Z0-9_]*"
identifier = ((~reserved + Regex(identifier_regex)).setName(
"identifier").setParseAction(lambda r: Identifier(r[0])))
identifier_keyword = (Regex(identifier_regex).setName(
"identifier keyword").setParseAction(lambda r: Identifier(r[0])))
# Types
variable_type = Or(types) | identifier
#############
# Grammar #
#############
expression = Forward().setName("expression")
statement = Forward().setName("statement")
#################
# Expressions #
#################
# Member access
def __build_recursive_member_access(expr, pos, result):
message += cr ^ lf ^ crlf
else:
message += crlf
message.leaveWhitespace()
build_message()
dispatcher.connect(build_message, 'parser.trailing_spaces', 'config')
dispatcher.connect(build_message, 'parser.soft_eol', 'config')
chanstring = charclass((0x01, 0x06), (0x08, 0x09), (0x0B, 0x0C), (0x0E, 0x1F),
(0x21, 0x2B), (0x2D, 0x39), (0x3B, 0xFF))
channelid = 5 * (charclass((0x41, 0x5A)) ^ digit)
channel = And([
Or([oneOf('# + &'), Literal('!') + Group(channelid)]),
Group(OneOrMore(chanstring)),
Optional(Suppress(Literal(':')) + Group(OneOrMore(chanstring)))
])
###
# Wildcard expressions
###
wildone = Literal('?')
wildmany = Literal('*')
nowild = charclass((0x01, 0x29), (0x2B, 0x3E), (0x40, 0xFF))
noesc = charclass((0x01, 0x5B), (0x5D, 0xFF))
mask = Optional(wildone ^ wildmany) + \
ZeroOrMore(nowild ^ (noesc + wildone) ^ (noesc + wildmany))
# Fall back to regex for parsing wildcards
IS = CaselessKeyword("is")
OR = CaselessKeyword("or")
AND = CaselessKeyword("and")
NOT = CaselessKeyword("not")
OF = CaselessKeyword("of")
EQUALS = Literal("=")
COLON = Literal(":")
L_C_BRKT = Literal("{")
R_C_BRKT = Literal("}")
L_BRKT = Literal("(")
R_BRKT = Literal(")")
COMMA = Literal(",")
BANG = Literal("!")
EQUALITY = Or((EQUALS, COLON, IS))
CODE = Literal("code")
SERIAL = Literal("serial")
TYPE = Literal("type")
COND = Or((Literal("condition"), Literal("cond")))
LABELLED = Literal("labelled")
PATH = Literal("path")
ASSY = Literal("assy")
_TRI_STATE_KEY_NAMES = ("development", "v-sense-move-1213",
"motor-rail-mod-1360", "tested", "has_headers",
"velcro-attached", "cased", "umbilical",
"climit_disabled", "dremel-mod", "tvs-mod-698",
"battery-aa-mod-1270")
TRI_STATE_KEYS = [Literal(x) for x in _TRI_STATE_KEY_NAMES]
def __repr__(self):
return 'CheckOrder(%r,%r,%r)' % (self.expr1, self.glyph, self.expr2)
@staticmethod
def parse_action(s, loc, tokens):
expr1 = tokens.get('expr1', None)
glyph = "".join(tokens['glyph'])
expr2 = tokens['expr2']
where = W(s, loc)
return CheckOrder(expr1, glyph, expr2, where=where)
comparisons_expr = {}
for glyph in CheckOrder.conditions:
if glyph == '!=':
# special case: ! must be followed by =
glyph_expression = Literal('!') - Literal('=')
glyph_expression.setName('!=')
else:
glyph_expression = Literal(glyph)
# 2015-05: not sure why this doesn't work and the alternative with + does
# expr = O(rvalue('expr1')) + glyph_expression('glyph') - rvalue('expr2')
expr = O(rvalue('expr1')) + glyph_expression('glyph') + rvalue('expr2')
expr.setParseAction(CheckOrder.parse_action)
add_contract(expr)
comparisons_expr[glyph] = expr
comparison_expr = Or(exprs=list(comparisons_expr.values()))
# PARSER
def ddlWord(string):
return WordStart(alphanums + "_") + CaselessLiteral(string) + WordEnd(
alphanums + "_")
# This function should be refactored if we find some database function which needs parameters
# Right now it works only for something like NOW() in MySQL default field value
def ddlFunctionWord(string):
return CaselessLiteral(string) + OneOrMore("(") + ZeroOrMore(
" ") + OneOrMore(")")
ddlString = Or([QuotedString("'"), QuotedString("\"",
escQuote='""'), QuotedString(
"`")])
negativeSign = Literal('-')
ddlNum = Combine(Optional(negativeSign) + Word(nums + "."))
ddlTerm = Word(alphanums + "_$")
ddlName = Or([ddlTerm, ddlString])
ddlMathOp = Word("+><=-")
ddlBoolean = Or([ddlWord("AND"), ddlWord("OR"), ddlWord("NOT")])
ddlArguments = "(" + delimitedList(Or([ddlString, ddlTerm, ddlNum])) + ")"
ddlMathCond = Optional("(") + delimitedList(
Or([
Group(ddlName + ddlMathOp + ddlName),
Group(ddlName + Optional(ddlWord("IS")) + ddlWord("NOT") + ddlWord(
"NULL")),
]),
delim=ddlBoolean) + Optional(")")
# http://en.wikipedia.org/wiki/Comparison_of_programming_languages_%28syntax%29#Libraries
# Python: import
# OpenSCAD: include (also use, but that's slightly different)
# C etc: #include
# LaTeX: \input, \include
# Java: import
# Modelica: import
# OK, let's go for import then...
importkeyword = Keyword("import").suppress()
# Constants
constantMap = {
"pi": Constant("pi", math.pi),
"e": Constant("e", math.e)
}
constant = Or([CaselessKeyword(constantName) for constantName in constantMap])
constant.setParseAction(lambda s, l, t: [constantMap[t[0]]])
# Parse numbers into floats straight away
fNumber.setParseAction(lambda s, l, t: [float(t[0])])
# Binary operators
binaryOperator = Word('+-*/^', exact=1) # N.B. To-the-power-of is ^ in this grammar at the moment - TODO make it accept Python syntax **
# Define mapping from binary operator symbols to composite expression classes
binaryOperatorMap = {
'+': SumExpression,
'-': DifferenceExpression,
'*': ProductExpression,
'/': QuotientExpression,
'^': PowerExpression
}
