def _build_grammar(self):
expr = Forward()
float_lit = Combine(Word(nums) + '.' + Word(nums))
float_lit.setName('float')
float_lit.setParseAction(lambda x: \
self.to_literal(float(x[0])))
int_lit = Word(nums)
int_lit.setName('int')
int_lit.setParseAction(lambda x: \
self.to_literal(int(x[0])))
num = (float_lit | int_lit)
num.setParseAction(lambda x: x[0])
tag_name = Word(alphas + "_", alphanums + "_")
tag_name.setName('tag_name')
tag_name.setParseAction(lambda t: tag_reference.TagReference(t[0]))
quoted_string = QuotedString("'")
quoted_string.setParseAction(lambda s: self.to_literal(s[0]))
oper = oneOf('+ * / -')
oper.setParseAction(lambda o: o[0])
lpar = Literal("(").suppress()
rpar = Literal(")").suppress()
arith = Group(lpar + expr + oper + expr + rpar)
arith.setParseAction(lambda t: \
self.to_arith(t[0][0], t[0][1], t[0][2]))
assign = tag_name + '=' + expr
assign.setName('assign')
assign.setParseAction(lambda x: self.to_assign(x[0],x[2]))
print_tags = Literal('?')
print_tags.setParseAction(lambda x: self.to_print_tags())
expr <<(arith|assign|tag_name|num|quoted_string|print_tags)
expr.setParseAction(lambda x: x[0])
return expr
def parse(self, string):
# An integer value
integer = Word(nums)
integer.setParseAction(Integer.parse)
integer.setName("integer")
# An expression in dice notation
expression = StringStart() + operatorPrecedence(integer, [
(CaselessLiteral('d').suppress(), 2, opAssoc.LEFT, Dice.parse_binary),
(CaselessLiteral('d').suppress(), 1, opAssoc.RIGHT, Dice.parse_unary),
(Literal('/').suppress(), 2, opAssoc.LEFT, Div.parse),
(Literal('*').suppress(), 2, opAssoc.LEFT, Mul.parse),
(Literal('-').suppress(), 2, opAssoc.LEFT, Sub.parse),
(Literal('+').suppress(), 2, opAssoc.LEFT, Add.parse),
(CaselessLiteral('t').suppress(), 1, opAssoc.LEFT, Total.parse),
(CaselessLiteral('s').suppress(), 1, opAssoc.LEFT, Sort.parse),
(Literal('^').suppress(), 2, opAssoc.LEFT, Keep.parse),
(Literal('v').suppress(), 2, opAssoc.LEFT, Drop.parse),
]) + StringEnd()
expression.setName("expression")
Keyword('masters').suppress() +
Optional(inet_ip_port_keyword_and_number_element) +
Optional(inet_dscp_port_keyword_and_number_element) - lbrack +
OneOrMore(zone_masters_set) + rbrack)('masters') + semicolon)
# pubkey number number number string; [ Zone ]
# The DNSSEC flags, protocol, and algorithm are specified, as well as a base-64 encoded string representing the key.
pubkey_flags = number_type
pubkey_protocol = number_type
pubkey_algorithm = number_type
# Secret are in base64 encoding scheme with 2-char paddings (RFC 1421)
# Handles up to 16K encoding
pubkey_secret_type = Word(
'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=',
max=16383)
pubkey_secret_type.setName('<key-secret>')
# secret <string>;
pubkey_secret = (
Combine(squote.suppress() + pubkey_secret_type + squote.suppress())
| Combine(dquote.suppress() + pubkey_secret_type + dquote.suppress()))
# pubkey is obsoleted since 9.0.0
zone_stmt_pubkey = (Keyword('pubkey').suppress() - Group(
pubkey_flags('flags') - pubkey_protocol('protocol') -
Optional(pubkey_algorithm('algorithms')) -
Optional(pubkey_secret('key_secret')))('pubkey') + semicolon)
# server-addresses server-addresses {ip-address; [...;]}; [ Zone ] [Def=N/A]
# Only used if zone type is 'static-stub'
zone_stmt_server_addresses = (
raise NotImplementedError
from pyparsing import Word, Literal, Optional, ZeroOrMore, Suppress, \
Group, Forward, OneOrMore, ParseException, \
CharsNotIn, Combine, StringStart, \
StringEnd, delimitedList
import string
filter_ = Forward()
attr = Word(
string.ascii_letters,
string.ascii_letters + string.digits + ';-',
)
attr.leaveWhitespace()
attr.setName('attr')
hexdigits = Word(string.hexdigits, exact=2)
hexdigits.setName('hexdigits')
escaped = Suppress(Literal('\\')) + hexdigits
escaped.setName('escaped')
def _p_escaped(s, l, t):
text = t[0]
return chr(int(text, 16))
escaped.setParseAction(_p_escaped)
value = Combine(OneOrMore(CharsNotIn('*()\\\0') | escaped))
value.setName('value')
equal = Literal("=")
# OPCODE 0b00001100
# OPCODE 0x0b
# OPCODE 'a'
# OPCODE 254-0x0a
# OPCODE 'a'&0b00001111
binbyte = Combine(Literal('0b') + Char('01') * 8)
binbyte.setName('binbyte')
binbyte.setParseAction(lambda t: [int(t[0], 2)])
hexbyte = Combine(Literal('0x') + Char(srange("[0-9a-fA-F]")) * 2)
hexbyte.setName('hexbyte')
hexbyte.setParseAction(lambda t: [int(t[0], 16)])
chrbyte = QuotedString(quoteChar="'", unquoteResults=True)
chrbyte.setName('char')
chrbyte.setParseAction(lambda t: [ord(t[0])])
number = Word(nums + '-')
number.setName('number')
number.setParseAction(lambda t: [int(t[0])])
allbytes = binbyte | hexbyte | chrbyte | number
mathtoken = Combine(oneOf('+ - & |') + allbytes)
bytemathexpression = Combine(allbytes + OneOrMore(mathtoken))
bytemathexpression.setParseAction(lambda t: [eval(t[0])])
byte = bytemathexpression | allbytes
byte.setName('byte')
# Words can be represented in binary, hex, label, or number (0-65535 or -32768-32767)
# OPCODE 0b0000111100001111
# OPCODE 0x2911
# OPCODE .label
# OPCODE .label+4
# OPCODE 2490
binword = Combine(Literal('0b') + Char('01') * 16)
binword.setName('binword')
# Set the parse action for the operator
if action is not None:
operator_expression.setParseAction(action)
this <<= operator_expression | last
last = this
# Set the full expression and return it
expression <<= last
return expression
# An integer value
integer = Word(nums)
integer.setParseAction(Integer.parse)
integer.setName("integer")
# An expression in dice notation
expression = (
StringStart()
+ operatorPrecedence(
integer,
[
(CaselessLiteral("d").suppress(), 2, opAssoc.LEFT, Dice.parse_binary),
(CaselessLiteral("d").suppress(), 1, opAssoc.RIGHT, Dice.parse_unary),
(Literal("/").suppress(), 2, opAssoc.LEFT, Div.parse),
(Literal("*").suppress(), 2, opAssoc.LEFT, Mul.parse),
(Literal("-").suppress(), 2, opAssoc.LEFT, Sub.parse),
(Literal("+").suppress(), 2, opAssoc.LEFT, Add.parse),
(CaselessLiteral("t").suppress(), 1, opAssoc.LEFT, Total.parse),
(CaselessLiteral("s").suppress(), 1, opAssoc.LEFT, Sort.parse),
from errors import TwillAssertionError, TwillNameError
from pyparsing import OneOrMore, Word, printables, quotedString, Optional, \
alphas, alphanums, ParseException, ZeroOrMore, restOfLine, Combine, \
Literal, Group, removeQuotes, CharsNotIn
import twill.commands as commands
import namespaces
import re
### pyparsing stuff
# basically, a valid Python identifier:
command = Word(alphas + "_", alphanums + "_")
command = command.setResultsName('command')
command.setName("command")
# arguments to it.
# we need to reimplement all this junk from pyparsing because pcre's
# idea of escapable characters contains a lot more than the C-like
# thing pyparsing implements
_bslash = "\\"
_sglQuote = Literal("'")
_dblQuote = Literal('"')
_escapables = printables
_escapedChar = Word(_bslash, _escapables, exact=2)
dblQuotedString = Combine( _dblQuote + ZeroOrMore( CharsNotIn('\\"\n\r') | _escapedChar | '""' ) + _dblQuote ).streamline().setName("string enclosed in double quotes")
sglQuotedString = Combine( _sglQuote + ZeroOrMore( CharsNotIn("\\'\n\r") | _escapedChar | "''" ) + _sglQuote ).streamline().setName("string enclosed in single quotes")
quotedArg = ( dblQuotedString | sglQuotedString )
quotedArg.setParseAction(removeQuotes)
def make_parser():
ParserElement.setDefaultWhitespaceChars(' \t')
EOL = OneOrMore(LineEnd()).suppress().setName("end of line")
Spaces = OneOrMore(" ").suppress()
# NOTE: These are not all 'printable' Unicode characters.
# If needed, expand the alphas_extra variable.
alphas_extra = ''.join(chr(x) for x in range(0x100, 0x350))
chars = printables + alphas8bit + alphas_extra
Token = Word(chars)
InlineComment = '#' - SkipTo(EOL)
WholelineComment = LineStart() + '#' - restOfLine - EOL
Argument = Token('arg').setName('argument')
Variable = Token('var').setName('variable')
KindObject = Keyword('kind')('object')
KindVerb = Keyword('is')('verb')
Kind = Named(Keyword('url') | Keyword('raw'))('arg')
MatchObject = Named(Keyword('arg'))('object')
data = Named(Keyword('data'))('object')
MatchVerb = Named(
Keyword('is') | Keyword('istype') | Keyword('matches')
| Keyword('rewrite'))('verb').setName('verb')
Pattern = Named(Group(OneOrMore(Spaces + Argument +
EOL)))('arg').leaveWhitespace()
ActionObject = Keyword('plumb')('object')
ActionVerb = Named(
Keyword('run') | Keyword('notify') | Keyword('download'))('verb')
Action = Named(originalTextFor(OneOrMore(Argument)))('arg')
ArgMatchClause = Group(MatchObject - MatchVerb - Variable - Pattern)
DataMatchClause = Group(data - MatchVerb - Pattern)
# Transform every 'data match' rule to an equivalent 'arg match' rule
def data_to_arg(toks):
assert (len(toks) == 1)
toks[0][0] = 'arg'
toks[0].insert(2, '{data}')
return toks
DataMatchClause.setParseAction(data_to_arg)
KindClause = Group(KindObject - KindVerb - Kind) - EOL
MatchClause = (DataMatchClause | ArgMatchClause)
ActionClause = Group(ActionObject - ActionVerb - Action) - EOL
MatchBlock = Group(ZeroOrMore(MatchClause('match-clause')))
ActionBlock = Group(OneOrMore(ActionClause('action-clause')))
# TODO: allow the excluded chars if they are escaped.
RuleName = Word(chars, excludeChars='{ } [ ]')('rule-name')
RuleHeading = Suppress('[') - RuleName - Suppress(']') - EOL
Rule = Group(RuleHeading - KindClause('kind-clause') -
MatchBlock('match-block') - ActionBlock('action-block'))
RulesFile = OneOrMore(Rule)
RulesFile.ignore(WholelineComment)
RulesFile.ignore(InlineComment)
for v in [MatchObject, ActionObject]:
v.setName('object')
for v in [MatchVerb, ActionVerb]:
v.setName('verb')
Kind.setName('kind')
data.setName('object')
Pattern.setName('pattern')
Action.setName('action or url')
KindClause.setName('kind clause')
MatchClause.setName('match clause')
ActionClause.setName('action clause')
MatchBlock.setName('match block')
ActionBlock.setName('action block')
Rule.setName('rule')
RuleName.setName('rule name')
RulesFile.setName('rules')
return RulesFile
def __init__(self):
self.stack = []
self.dice_roles = []
self.binary_ops = {
'+': operator.add,
'-': operator.sub,
'*': operator.mul,
'/': operator.truediv,
'^': operator.pow,
'>': operator.gt,
'>=': operator.ge,
'<': operator.lt,
'<=': operator.le,
'!=': operator.ne,
'==': operator.eq,
'or': operator.or_,
'and': operator.and_
}
self.constants = {'PI': math.pi, 'E': math.e}
self.functions = {
'sum': lambda *a: sum(a),
'sin': math.sin,
'cos': math.cos,
'tan': math.tan,
'exp': math.exp,
'hypot': math.hypot,
'abs': abs,
'trunc': int,
'round': round,
'sgn': lambda a: -1 if a < -math.e else 1 if a > math.e else 0,
'multiply': lambda a, b: a * b,
'all': lambda *a: all(a),
'any': lambda *a: any(a)
}
# lang vars
e = CaselessKeyword("E")
pi = CaselessKeyword("PI")
number = Regex(r"[+-]?\d+(?:\.\d*)?(?:[eE][+-]?\d+)?")
number.setName('Number')
ident = Word(alphas, alphanums + "_$")
ident.setName('Ident')
dice = Regex(r'\d?[dD]\d+')
dice.setName('Dice')
plus, minus, lt, le, gt, ge, eq, ne, or_, and_ = map(
Literal, ['+', '-', '<', '<=', '>', '>=', '==', '!=', 'or', 'and'])
bi_op = plus | minus | lt | le | gt | ge | eq | ne | or_ | and_
bi_op.setName('LowBinaryOp')
mult = Literal('*')
div = Literal('/')
multop = mult | div
multop.setName('MediumBinaryOp')
expop = Literal('^')
expop.setName('HighBinaryOp')
lpar = Suppress('(')
rpar = Suppress(')')
factor = Forward()
expr = Forward()
expr_list = delimitedList(Group(expr))
expr_list.setName('ExpressionList')
def dice_role(s: str) -> int:
rolls = DiceRolls(roll=s, results=[])
s = s.lower()
if s.startswith('d'):
count = 1
limit = s[1:]
else:
count, limit = s.lower().split('d')
count = int(count)
limit = int(limit)
for _ in range(0, count):
roll = random.randint(1, limit)
rolls.results.append(roll)
self.dice_roles.append(rolls)
return rolls.sum
def insert_fn_arg_count_tuple(t: Tuple) -> None:
fn = t.pop(0)
argc = len(t[0])
t.insert(0, (fn, argc))
def push(tokens) -> None:
self.stack.append(tokens[0])
def push_unary_minus(tokens) -> None:
if '-' in tokens:
push('unary -')
def push_dice(t: ParseResults) -> None:
self.stack.append(functools.partial(dice_role, t[0]))
dice.setParseAction(push_dice)
fn_call = ((ident + lpar - Group(expr_list) +
rpar).setParseAction(insert_fn_arg_count_tuple))
atom = dice | (bi_op[...] + (
((fn_call | pi | e | number | ident).setParseAction(push))
| Group(lpar + expr + rpar)).setParseAction(push_unary_minus))
factor <<= atom + (expop + factor).setParseAction(push)[...]
term = factor + (multop + factor).setParseAction(push)[...]
expr <<= term + (bi_op + term).setParseAction(push)[...]
self.expr = expr
expr.setName('Expression')
factor.setName('Factor')
atom.setName('Atom')
term.setName('Term')
from io import StringIO
from pyparsing import Word, printables, Optional, \
alphas, alphanums, ZeroOrMore, restOfLine, Combine, \
Literal, Group, removeQuotes, CharsNotIn
import twill.commands as commands
from . import namespaces
from .errors import TwillAssertionError, TwillNameError
# pyparsing stuff
# basically, a valid Python identifier:
command = Word(alphas + "_", alphanums + "_")
command = command.setResultsName('command')
command.setName("command")
# arguments to it.
# we need to reimplement all this junk from pyparsing because pcre's
# idea of escapable characters contains a lot more than the C-like
# thing pyparsing implements
_bslash = "\\"
_sglQuote = Literal("'")
_dblQuote = Literal('"')
_escapables = printables
_escapedChar = Word(_bslash, _escapables, exact=2)
dblQuotedString = Combine(
_dblQuote + ZeroOrMore(CharsNotIn('\\"\n\r') | _escapedChar | '""') +
_dblQuote).streamline().setName("string enclosed in double quotes")
sglQuotedString = Combine(
domain_generic_fqdn = Combine(domain_generic_label +
ZeroOrMore(Literal('.') + domain_generic_label) +
Optional(Char('.')))
domain_generic_fqdn.setName('<generic-fqdn>')
domain_generic_fqdn.setResultsName('domain_name')
rr_domain_name = Combine(domain_generic_fqdn + Optional(Literal('.')))
rr_domain_name.setName('<rr_domain_name>')
charset_acl_name_base = alphanums + '_-.+~@$%^&*()=[]\\|:<>`?' # no semicolon nor curly braces allowed
charset_view_name_base = alphanums + '_-.+~@$%^&*()=[]\\|:<>`?' # no semicolon nor curly braces allowed
charset_view_name_dquotable = charset_view_name_base + "\'"
charset_view_name_squotable = charset_view_name_base + '\"'
view_name_base = Word(charset_acl_name_base, max=64)
view_name_base.setName('<view-name-unquoted>')
view_name_dquotable = Combine(
Char('"') + Word(charset_view_name_dquotable, max=62) + Char('"'))
view_name_squotable = Combine(
Char("'") + Word(charset_view_name_squotable, max=62) + Char("'"))
view_name = (view_name_dquotable(None)
| view_name_squotable(None)
| view_name_base(None))('view_name')
view_name.setName('<view-name>')
charset_filename_base = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!#$%&()*+,-.:?@[\\]^_`|~="
charset_filename_base_quotable = charset_filename_base + ';' + '{}'
# only if quoted, can have a space character in its filename
def integer():
token = Word(nums)
token.setParseAction(Integer.parse)
token.setName("integer")
return token
# keyname_dquoted.setName('keyname_squoted')
trusted_key_domain_name = Group(
trusted_keyname_dquoted
| trusted_keyname_squoted
| trusted_keyname_type
)
trusted_key_flags_type = number_type('flags')
trusted_key_flags_type.setName('<key-flags-id>')
trusted_key_protocol_type = number_type('protocol_id')
trusted_key_protocol_type.setName('<key-protocol-id>')
trusted_key_algorithm_name = Word(alphanums + '-')('algorithm')
trusted_key_algorithm_name.setName('<key-algorithm>')
trusted_key_algorithm_type = number_type('algorithm_id')
trusted_key_algorithm_type.setName('<key-algorithm-id>')
# Secret are in base64 encoding scheme with 2-char paddings (RFC 1421)
# Handles up to 16K encoding
charset_key_secret_base = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/='
charset_key_secret_base_squote_allowed = charset_key_secret_base + "'"
charset_key_secret_base_dquote_allowed = charset_key_secret_base + '"'
quoted_trusted_key_secret_type = (
Combine(squote + Word(charset_key_secret_base_dquote_allowed) + squote)
| Combine(dquote + Word(charset_key_secret_base_squote_allowed) + dquote)
)
LBRACKET = L("[").suppress()
RBRACKET = L("]").suppress()
COLON = L(":").suppress()
NIL = L('"-"')
NIL.setName("Nil")
NIL.setParseAction(lambda s, l, t: NilValue)
PRIORITY = LANGLE + Word(srange("[0-9]"), min=1, max=3) + RANGLE # 191 Max
PRIORITY = PRIORITY.setResultsName("priority")
PRIORITY.setName("Priority")
PRIORITY.setParseAction(lambda s, l, t: int(t[0]))
TIMESTAMP = Word(printables)
TIMESTAMP = TIMESTAMP.setResultsName("timestamp")
TIMESTAMP.setName("Timestamp")
HOSTNAME = Combine(NIL | Word(printables))
HOSTNAME = HOSTNAME.setResultsName("hostname")
HOSTNAME.setName("Hostname")
APPNAME = Word("".join(set(printables) - {"["}))
APPNAME = APPNAME.setResultsName("appname")
APPNAME.setName("AppName")
PROCID = Combine(LBRACKET + Word("".join(set(printables) - {"]"})) + RBRACKET)
PROCID = PROCID.setResultsName("procid")
PROCID.setName("ProcID")
HEADER = PRIORITY + TIMESTAMP + SP + HOSTNAME + SP + APPNAME + PROCID
Clause: keys
Title: Clause statement for key
Description: Provides key-related grammar in PyParsing engine
for ISC-configuration style
"""
from pyparsing import Word, alphanums, Group, Keyword, ZeroOrMore
from bind9_parser.isc_utils import semicolon, lbrack, rbrack, key_id, key_secret
# NOTE: If any declaration here is to be used OUTSIDE of the 'keys' clause,
# it should instead be defined in isc_utils.py
key_algorithm_name = Word(alphanums + '-')('algorithm')
key_algorithm_name.setName('<key-algorithm>')
# algorithm <string>;
key_algorithm_element = (Keyword('algorithm').suppress() -
key_algorithm_name('algorithm') + semicolon)
key_algorithm_element.setName('algorithm <key-algorithm>;')
# secret <key_secret>;
key_secret_element = (Keyword('secret').suppress() - key_secret('secret') +
semicolon)
key_secret_element.setName('secret <key_secret>;')
# key <key-name> { algorithm <string>; secret <key-secret>; };
# key key_id {
# algorithm algorithm_id;
# secret secret_string;
def create_parser(config, constants):
EXPRESSION = pyparsing.Forward()
VARIABLE = Word(pyparsing.alphas + '_', pyparsing.alphanums + '_-')
VARIABLE.setName('Variable')
VARIABLE.setParseAction(
add_logging(lambda string, location, result: Variable(result[0])))
REAL = pyparsing_common.real
REAL.setParseAction(
add_logging(
lambda string, location, result: UnnamedConstant(result[0])))
SCI_REAL = pyparsing_common.sci_real
SCI_REAL.setParseAction(
add_logging(
lambda string, location, result: UnnamedConstant(result[0])))
SIGNED_INTEGER = pyparsing_common.signed_integer
SIGNED_INTEGER.setParseAction(
add_logging(
lambda string, location, result: UnnamedConstant(result[0])))
NUMBER = pyparsing.Or([REAL, SCI_REAL, SIGNED_INTEGER])
COMPONENT = pyparsing.Or(
[f.get_parser(EXPRESSION)
for f in config if isinstance(f, PFunction)] + [
p for p in map(lambda c: c.get_parser(EXPRESSION), constants)
if p is not None
] + [NUMBER, VARIABLE])
# TODO Generating operators_config should be rewritten
operators = defaultdict(list)
for operator in config:
if not isinstance(operator, POperator):
continue
operators[operator.precedence].append(operator)
operators_config = []
for precedence, ops in sorted(operators.items()):
assert all(ops[0].rhs_only == o.rhs_only for o in ops), ops
# TODO This is a hack, is there a nicer way?
from .identifiers import IDs
if ops[0].id in (IDs.MINUS, IDs.PLUS):
assert ops[0]._rhs_only
parser = pyparsing.Or(
[Literal(o.op) + ~pyparsing.FollowedBy(NUMBER) for o in ops])
elif ops[0].id in (IDs.SQUARE, ):
assert ops[0]._lhs_only
parser = pyparsing.Or(
[Literal(o.op) + ~pyparsing.FollowedBy(NUMBER) for o in ops])
else:
parser = pyparsing.Or([Literal(o.op) for o in ops])
if ops[0].rhs_only:
def parse_action(string,
location,
result,
op_map={o.op: o
for o in ops}):
assert len(result) == 1, result
result = result[0]
assert len(result) == 2, result
return op_map[result[0]](result[1])
operators_config.append((parser, 1, opAssoc.RIGHT, parse_action))
elif ops[0].lhs_only:
def parse_action(string,
location,
result,
op_map={o.op: o
for o in ops}):
assert len(result) == 1, result
result = result[0]
assert len(result) == 2, result
return op_map[result[0]](result[1])
operators_config.append((parser, 1, opAssoc.LEFT, parse_action))
else:
def parse_action(string,
location,
result,
op_map={o.op: o
for o in ops}):
assert len(result) == 1, result
result = result[0]
expression = result[0]
expression_args = [result[2]]
last_op_name = result[1]
for op_name, value in zip(result[3::2], result[4::2]):
if op_name == last_op_name:
expression_args.append(value)
else:
expression = Expression(op_map[last_op_name].id,
expression, *expression_args)
expression_args = [value]
last_op_name = op_name
expression = Expression(op_map[last_op_name].id, expression,
*expression_args)
# for operator, value in zip(result[1::2], result[2::2]):
# operator = op_map[operator]
# expression = Expression(operator.id, expression, value)
return expression
operators_config.append((parser, 2, opAssoc.LEFT, parse_action))
EXPRESSION << pyparsing.infixNotation(COMPONENT, operators_config)
return EXPRESSION
# Set the parse action for the operator
if action is not None:
operator_expression.setParseAction(action)
this <<= (operator_expression | last)
last = this
# Set the full expression and return it
expression <<= last
return expression
# An integer value
integer = Word(nums)
integer.setParseAction(Integer.parse)
integer.setName("integer")
# An expression in dice notation
expression = StringStart() + operatorPrecedence(integer, [
(CaselessLiteral('d').suppress(), 2, opAssoc.LEFT, Dice.parse_binary),
(CaselessLiteral('d').suppress(), 1, opAssoc.RIGHT, Dice.parse_unary),
(Literal('/').suppress(), 2, opAssoc.LEFT, Div.parse),
(Literal('*').suppress(), 2, opAssoc.LEFT, Mul.parse),
(Literal('-').suppress(), 2, opAssoc.LEFT, Sub.parse),
(Literal('+').suppress(), 2, opAssoc.LEFT, Add.parse),
(Word('+-').suppress(), 1, opAssoc.RIGHT, AddEvenSubOdd.parse),
(Word('+-').suppress(), 2, opAssoc.LEFT, AddEvenSubOdd.parse),
(CaselessLiteral('t').suppress(), 1, opAssoc.LEFT, Total.parse),
(CaselessLiteral('s').suppress(), 1, opAssoc.LEFT, Sort.parse),
(Literal('^').suppress(), 2, opAssoc.LEFT, Keep.parse),
(Literal('v').suppress(), 2, opAssoc.LEFT, Drop.parse),
NullValue = NullValue()
printables = "".join(set(_printables + " " + "\t") - {"|", "@"})
PIPE = L("|").suppress()
AT = L("@").suppress()
NULL = L("(null)")
NULL.setParseAction(lambda s, l, t: NullValue)
TIMESTAMP = Word(printables)
TIMESTAMP = TIMESTAMP.setResultsName("timestamp")
TIMESTAMP.setName("Timestamp")
COUNTRY_CODE = Word(printables)
COUNTRY_CODE = COUNTRY_CODE.setResultsName("country_code")
COUNTRY_CODE.setName("Country Code")
URL = Word(printables)
URL = URL.setResultsName("url")
URL.setName("URL")
REQUEST = TIMESTAMP + PIPE + Optional(COUNTRY_CODE) + PIPE + URL
PROJECT_NAME = NULL | Word(printables)
PROJECT_NAME = PROJECT_NAME.setResultsName("project_name")
PROJECT_NAME.setName("Project Name")
print("IPv4 subnet is out of range: %d" % value)
print("strg: %s" % strg)
print("loc: %s" % loc)
return None
dotted_decimal = Combine(
Word(nums, max=3) + Literal('.') + Word(nums, max=3) + Literal('.') +
Word(nums, max=3) + Literal('.') + Word(nums, max=3))
# Bind9 naming convention
ip4_addr = pyparsing_common.ipv4_address
ip4_addr.setName('<ip4_addr>')
ip4s_subnet = Word(nums, min=1, max=2)
ip4s_subnet.setName('<ip4_or_ip4_subnet>')
ip4s_prefix = Combine(ip4_addr + '/' - ip4s_subnet)
ip4s_prefix.setName('<ip4subnet>')
# Device Index (aka Unix sin6_scope_id) can be 32-bit integer or 64-char readable device name
# _ip6_device_index = r'%([0-9]{1,10})|([a-zA-Z0-9\.\-_]{1,64})'
_ip6_device_index = r'%' + \
Combine(
Word(nums, min=1, max=10)
| Word(alphanums, min=1, max=63)
)
# Apparently, pyparsing_common.ipv6_address cannot the followingz:
# - do device index suffix of "%eth0" or "%1"
# - Support IPv4 notation after short or mixed IPv6
# so we roll our own IPv6 parser
# isc_file_name has no '/', (but path_name do)
charset_filename_base = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!#$%&()*+,-.:?@[\\]^_`|~="
# charset_filename_base is largely printable, but has
# no double-quote, single-quote, semicolon, curly-braces, nor slash
# no dquote/squote/slash/space
charset_filename_base_quotable = charset_filename_base + ';' + '{}'
# has single-quote, semicolon, and space
charset_filename_has_squote = charset_filename_base_quotable + "' "
# has double-quote, semicolon, and space
charset_filename_has_dquote = charset_filename_base_quotable + '" '
filename_base = Word(charset_filename_base)
filename_base.setName('<printable-chars_has_no_squote_dquote_semicolon_slash_space>')
filename_dquotable = Combine(
dquote
+ Word(charset_filename_has_squote)
+ dquote
) # inverse quote types here
filename_squotable = Combine(
squote
+ Word(charset_filename_has_dquote)
+ squote
) # inverse quote types here
filename_dquotable.setName('<printable-chars_has_no_dquote_slash>')
isc_file_name = (
LBRACKET = L("[").suppress()
RBRACKET = L("]").suppress()
COLON = L(":").suppress()
NIL = L('"-"')
NIL.setName("Nil")
NIL.setParseAction(lambda s, l, t: NilValue)
PRIORITY = LANGLE + Word(srange("[0-9]"), min=1, max=3) + RANGLE # 191 Max
PRIORITY = PRIORITY.setResultsName("priority")
PRIORITY.setName("Priority")
PRIORITY.setParseAction(lambda s, l, t: int(t[0]))
TIMESTAMP = Word(printables)
TIMESTAMP = TIMESTAMP.setResultsName("timestamp")
TIMESTAMP.setName("Timestamp")
HOSTNAME = Combine(NIL | Word(printables))
HOSTNAME = HOSTNAME.setResultsName("hostname")
HOSTNAME.setName("Hostname")
APPNAME = Word("".join(set(printables) - {"["}))
APPNAME = APPNAME.setResultsName("appname")
APPNAME.setName("AppName")
PROCID = Combine(LBRACKET + Word("".join(set(printables) - {"]"})) + RBRACKET)
PROCID = PROCID.setResultsName("procid")
PROCID.setName("ProcID")
HEADER = PRIORITY + TIMESTAMP + SP + HOSTNAME + SP + APPNAME + PROCID
def build(self):
# ------------------------------------------
# C. building blocks
# ------------------------------------------
self.termop = Regex( "|".join(self.neighbourhood_symbols), re.IGNORECASE ).setParseAction( upcaseTokens ).setName("termop")
termword = Word(self.unicode_printables + self.separators + self.wildcards).setName("term")
termword_termop = (termword + OneOrMore( self.termop + termword ))
# ------------------------------------------
# D. triple
# ------------------------------------------
index = Word(alphanums).setName("index")
#index = Word(indexchars).setName("index")
#SolrProximitySuffix = Suppress(Optional(Word('~') + Word(nums)))
binop = oneOf(self.binop_symbols, caseless=True).setName("binop")
term = (
# Attempt to parse {!complexphrase}text:"((aussto* OR eject* OR pusher*) AND (verriegel* OR lock* OR sperr*))"~6 ...
# ... but failed.
#Combine(quotedString.setParseAction(removeQuotes) + SolrProximitySuffix).setName("term") ^
# term is a quoted string, easy peasy
quotedString.setName("term") ^
# term is just a termword, easy too
termword.setName("term") ^
# term contains neighbourhood operators, so should have been wrapped in parenthesis
Combine('(' + Suppress(ZeroOrMore(' ')) + termword_termop + Suppress(ZeroOrMore(' ')) + ')').setName("term") ^
# convenience/gracefulness: we also allow terms containing
# neighbourhood operators without being wrapped in parenthesis
Combine(termword_termop).setName("term")
)
# ------------------------------------------
# E. condition
# ------------------------------------------
cqlStatement = Forward()
# Parse regular cql condition notation 'index=term'.
cqlConditionBase = Group(
# a regular triple
( index + binop + term ).setResultsName("triple") |
# a regular subquery
( "(" + cqlStatement + ")" ).setResultsName("subquery")
)
# Parse value shortcut notations like 'index=(term)' or 'index=(term1 and term2 or term3)'.
cqlConditionShortcut = Group(
# a triple in value shortcut notation (contains only the single term)
# "term + NotAny(binop)" helps giving proper error messages like
# "ParseException: Expected term (at char 4)" for erroneous queries like "foo="
( term + NotAny(binop) ).setResultsName("triple-short") |
# a subquery containing values in shortcut notation
( index + binop + "(" + cqlStatement + ")" ).setResultsName("subquery-short")
)
#cqlCondition = cqlConditionBase
cqlCondition = cqlConditionBase | cqlConditionShortcut
# ------------------------------------------
# F. statement
# ------------------------------------------
cqlStatement << cqlCondition + ZeroOrMore( self.booleans_or + cqlStatement )
# apply SQL comment format
cqlComment = "--" + restOfLine
cqlStatement.ignore(cqlComment)
self.parser = cqlStatement
raise NotImplementedError()
from pyparsing import Word, Literal, Optional, ZeroOrMore, Suppress, \
Group, Forward, OneOrMore, ParseException, \
CharsNotIn, Combine, StringStart, \
StringEnd, delimitedList
import string
filter_ = Forward()
attr = Word(string.ascii_letters,
string.ascii_letters + string.digits + ';-',)
attr.leaveWhitespace()
attr.setName('attr')
hexdigits = Word(string.hexdigits, exact=2)
hexdigits.setName('hexdigits')
escaped = Suppress(Literal('\\')) + hexdigits
escaped.setName('escaped')
def _p_escaped(s, l, t):
text = t[0]
return chr(int(text, 16))
escaped.setParseAction(_p_escaped)
value = Combine(OneOrMore(CharsNotIn('*()\\\0') | escaped))
value.setName('value')
equal = Literal("=")
managed_key_domain_name = (
managed_keyname_dquoted
| managed_keyname_squoted
| managed_keyname_type
)('rr_domain')
managed_key_type = Keyword('initial-key') # Future will have multiple options
managed_key_flags_type = number_type('flags')
managed_key_flags_type.setName('<key-flags-id>')
managed_key_protocol_type = number_type('protocol_id')
managed_key_protocol_type.setName('<key-protocol-id>')
managed_key_algorithm_name = Word(alphanums + '-')('algorithm_type')
managed_key_algorithm_name.setName('<key-algorithm>')
managed_key_algorithm_type = number_type('algorithm_id')
managed_key_algorithm_type.setName('<key-algorithm-id>')
# Secret are in base64 encoding scheme with 2-char paddings (RFC 1421)
# Handles up to 16K encoding
charset_key_secret_base = ' \n\r\tABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/='
charset_key_secret_base_squote_allowed = charset_key_secret_base + "'"
charset_key_secret_base_dquote_allowed = charset_key_secret_base + '"'
managed_key_secret_type = (
Combine(squote + Word(charset_key_secret_base_dquote_allowed) + squote)
| Combine(dquote + Word(charset_key_secret_base_squote_allowed) + dquote)
| Word(charset_key_secret_base)
)('key_secret')
ZeroOrMore, restOfLine
import raco
from raco import expression
import raco.datalog.model as model
def show(x):
print x
return x
drop = lambda x: Literal(x).suppress()
# define Datalog tokens
ident = Word(alphas, alphanums + "_$")
predicate = ident.setName("Predicate")
E = CaselessLiteral("E")
# Get all the aggregate expression classes
aggregate_functions = raco.expression.aggregate_functions()
# All binary operators
binopclasses = expression.binary_ops()
# a list of string literals representing opcodes
opcodes = sum([oc.literals for oc in binopclasses], [])
binopstr = " ".join(opcodes)
INTO = Keyword("into", caseless=True)
VALUES = Keyword("values", caseless=True)
UPDATE = Keyword("update", caseless=True)
SET = Keyword("set", caseless=True)
IF = Keyword("if", caseless=True)
NOT = Keyword("not", caseless=True)
EXISTS = Keyword("exists", caseless=True)
DELETE = Keyword("delete", caseless=True)
USE = Keyword("use", caseless=True)
CREATE = Keyword("create", caseless=True)
TABLE = Keyword("table", caseless=True)
PRIMARY = Keyword("primary", caseless=True)
KEY = Keyword("key", caseless=True)
# column names
columnName = ident.setName("column").addParseAction(downcaseTokens)
columnNameList = Group(delimitedList(columnName))
# table name
keyspaceName = ident.addParseAction(downcaseTokens).setName("keyspace")
tableName = Group(
Optional(keyspaceName + ".") +
ident.addParseAction(downcaseTokens)).setName("table")
# where clause
and_ = Keyword("and", caseless=True)
binop = oneOf("= eq != < > >= <= eq ne lt le gt ge", caseless=True)
Rval = realNum('real') | intNum('int') | quotedString(
'quoted') # need to add support for alg expressions
RvalList = Group(delimitedList(Rval))
