def build_parser():
"""
Build a pyparsing parser for our custom topology description language.
:rtype: pyparsing.MatchFirst
:return: A pyparsing parser.
"""
from pyparsing import (Word, Literal, QuotedString, StringStart, StringEnd,
alphas, nums, alphanums, Group, OneOrMore, Optional)
number = Word(nums)
text = QuotedString('"')
identifier = Word(alphas, alphanums + '_')
attribute = (identifier('key') + Literal('=') +
(text | number | identifier)('value'))
attributes = (Literal('[') + OneOrMore(Group(attribute))('attributes') +
Literal(']'))
node = identifier('node')
port = node + Literal(':') + (identifier | number)('port')
link = port('endpoint_a') + Literal('--') + port('endpoint_b')
nodes_spec = (StringStart() + Optional(attributes) +
OneOrMore(Group(node))('nodes') + StringEnd())
ports_spec = (StringStart() + Optional(attributes) +
OneOrMore(Group(port))('ports') + StringEnd())
link_spec = (StringStart() + Optional(attributes) + link('link') +
StringEnd())
statement = link_spec | ports_spec | nodes_spec
return statement
def dom_document_setup():
crn_DWC = "".join(
[x for x in ParseElementEnhance.DEFAULT_WHITE_CHARS if x != "\n"])
ParseElementEnhance.setDefaultWhitespaceChars(crn_DWC)
W = Word
G = Group
S = Suppress
O = Optional
L = Literal
identifier = W(alphas, alphanums + "_")
number = W(nums, nums)
domain = G(OneOrMore((G(identifier + O("*")) | "?" | "+")))
dotparen = G(OneOrMore(W("().+", max=1)))
structure = domain + OneOrMore(LineEnd().suppress()) + dotparen
sequence = G(
S("sequence") + identifier + S(":") + number +
OneOrMore(LineEnd().suppress()))
molecule = G(identifier + S(":") + OneOrMore(LineEnd().suppress()) +
structure)
document = StringStart() + ZeroOrMore(LineEnd().suppress()) + G(
ZeroOrMore(sequence)) + G(
OneOrMore(molecule +
OneOrMore(LineEnd().suppress()))) + StringEnd()
document.ignore(pythonStyleComment)
return document
def term():
""" """
token = (
StringStart() + operand() + StringEnd()
^ StringStart() + Group(operand() + operator() + operand()) + StringEnd()
)
token.setName("term")
return token
def _parse_cut_ami(text):
"""Parse "cut ami" command using pyparsing"""
# Word == single token
edctoken = Word(alphanums + '_')
withtoken = Word(printables.replace('=', ''))
preamble = Suppress(Literal('cut') + 'ami')
# e.g. prod-edx-exdapp. Combining into 1 token enforces lack of whitespace
e_d_c = Combine(
edctoken('environment') + '-' + edctoken('deployment') + '-' +
edctoken('cluster'))
# e.g. cut ami for prod-edx-edxapp. Subsequent string literals are converted when added to a pyparsing object.
for_from = Suppress('for') + e_d_c('for_edc') + Suppress(
'from') + e_d_c('from_edc')
# e.g. with foo=bar bing=baz.
# Group puts the k=v pairs in sublists instead of flattening them to the top-level token list.
with_stmt = Suppress('with')
with_stmt += OneOrMore(
Group(withtoken('key') + Suppress('=') +
withtoken('value')))('overrides')
# e.g. using ami-deadbeef
using_stmt = Suppress('using') + Regex('ami-[0-9a-f]{8}')('ami_id')
# 0-1 with and using clauses in any order (see Each())
modifiers = Optional(with_stmt('with_stmt')) & Optional(
using_stmt('using_stmt'))
# 0-1 verbose and noop options in any order (as above)
options = Optional(Literal('verbose')('verbose')) & Optional(
Literal('noop')('noop'))
pattern = StringStart(
) + preamble + options + for_from + modifiers + StringEnd()
parsed = pattern.parseString(text)
return {
'dest_env': parsed.for_edc.environment,
'dest_dep': parsed.for_edc.deployment,
'dest_play': parsed.for_edc.cluster,
'source_env': parsed.from_edc.environment,
'source_dep': parsed.from_edc.deployment,
'source_play': parsed.from_edc.cluster,
'base_ami':
parsed.using_stmt.ami_id if parsed.using_stmt else None,
'version_overrides':
{i.key: i.value
for i in parsed.with_stmt.overrides}
if parsed.with_stmt else None,
'verbose': bool(parsed.verbose),
'noop': bool(parsed.noop),
}
def term():
"""
"""
token = StringStart() + dice() + StringEnd() \
| StringStart() + dice() + operator() + integer() + StringEnd() \
| StringStart() + dice() + flags() + StringEnd() \
| StringStart() + dice() + flags() + operator() + integer() + StringEnd()
token.setName("term")
token.setResultsName("term")
return token
def _compile_grammar(self):
# type: () -> ParserElement
"""
Takes the individual grammars from each registered directive and compiles them into a full test fixture grammar
whose callback methods are the bound methods on this class instance.
:return: The full PyParsing grammar for test fixture files.
"""
grammars = [
(LineEnd().suppress()).setParseAction(
functools.partial(self._finalize_test_case)
)
]
# directives
for directive_class in get_all_directives():
grammars.append(
LineStart() +
directive_class.get_full_grammar().setParseAction(
functools.partial(self._ingest_directive, directive_class)
) +
LineEnd()
)
return StringStart() + OneOrMore(MatchFirst(grammars)) + StringEnd()
def wildcard(str):
parsed = parse(str, mask)
if not parsed:
return parsed
return StringStart() + \
Regex(''.join([wild_to_match(x) for x in parsed])) + \
StringEnd()
def expr(self) -> ParserElement:
return (StringStart() | PrecededBy(
Regex(r"\W", flags=re.UNICODE), retreat=1)) + Combine(
"!" + Word(printables + " ", min=3,
excludeChars="|!").setResultsName("url") +
Optional("|") + SkipTo("!", failOn="\n") +
"!", ).setParseAction(self.action)
def create_arg_parser():
from pyparsing import Literal, Word, delimitedList, Group, \
StringStart, StringEnd, Optional, nums, alphas, alphanums
ident = Word(alphas, alphanums + "_")
inumber = Word("+-" + nums, nums)
history = Optional(Literal('[').suppress() + inumber +
Literal(']').suppress(),
default=0)("history")
history.setParseAction(lambda str, loc, toks: int(toks[0]))
variable = Group(Word(alphas, alphanums + '._') + history)
derivative = Group(Literal('d') + variable\
+ Literal('/').suppress() + Literal('dt'))
trace = Group(
Literal('tr') + Literal('(').suppress() +
Optional(ident + Literal(',').suppress(), default=None) + variable +
Literal(')').suppress())
generalized_var = derivative | trace | variable
args = StringStart() + delimitedList(generalized_var) + StringEnd()
return args
def _define_vs():
KEY = Word(alphas + '_$', alphanums +
'_$').setName('identifier').setResultsName('key') # noqa
VALUE = originalTextFor(_define_json()).setResultsName('value')
# validator name, eg: int
NAME = Optional(
Optional(Suppress('?')) +
pyparsing_common.identifier.setResultsName('name')) # noqa
# refers, eg: @xx@yy
REFERS = Group(ZeroOrMore(Suppress('@') +
pyparsing_common.identifier)).setResultsName(
'refers') # noqa
# args, eg: (), (1), (1,2,3), ([1,2], {"key":"value"}, "Any JSON")
ARGS = Group(
Optional(
Suppress('(') + Optional(delimitedList(VALUE)) +
Suppress(')'))).setResultsName('args') # noqa
# key-value, eg: key, key=True, key=[1,2,3]
KW = Group(KEY + Optional(Suppress('=') + VALUE))
# kwargs, eg: &key1&key2=True&key3=[1,2,3]
KWARGS = Group(ZeroOrMore(Suppress('&') + KW)).setResultsName('kwargs')
# lead xxx is key: xxx@yyy, xxx?yyy, $self&abc
# lead xxx except '$self' is validator name: xxx(1,2), xxx&abc, xxx
SELF = Literal('$self').setResultsName('key')
VS_KEY = Optional((KEY + FollowedBy(Word('@?'))) | SELF)
VS_DEF = REFERS + NAME + ARGS + KWARGS
return StringStart() + VS_KEY + VS_DEF + StringEnd()
def create_bnf(term_descs):
"""term_descs .. list of TermParse objects
(sign, term_name, term_arg_names), where sign can be real or complex
multiplier"""
lc = ['+'] # Linear combination context.
equal = Literal("=").setParseAction(rhs(lc))
zero = Literal("0").suppress()
point = Literal(".")
e = CaselessLiteral("E")
inumber = Word("+-" + nums, nums)
fnumber = Combine(Word("+-" + nums, nums) +
Optional(point + Optional(Word(nums))) +
Optional(e + Word("+-" + nums, nums)))
number = fnumber + Optional(Literal('j'), default='')
add_op = oneOf('+ -')
number_expr = Forward()
number_expr << ZeroOrMore('(') + number \
+ ZeroOrMore(add_op + number_expr) \
+ ZeroOrMore(')')
ident = Word(alphas, alphanums + "_")
integral = Combine((Literal('i') + Word(alphanums)) | Literal('i')
| Literal('a') | Word(nums))("integral")
history = Optional('[' + inumber + ']', default='')("history")
variable = Combine(Word(alphas, alphanums + '._') + history)
derivative = Combine(Literal('d') + variable \
+ Literal('/') + Literal('dt'))
trace = Combine(Literal('tr') + '(' + variable + ')')
generalized_var = derivative | trace | variable
args = Group(delimitedList(generalized_var))
flag = Literal('a')
term = Optional(Literal('+') | Literal('-'), default='+')("sign") \
+ Optional(number_expr + Literal('*').suppress(),
default=['1.0', ''])("mul") \
+ Combine(ident("name") \
+ Optional("." + (integral + "."
+ ident("region") + "." + flag("flag") |
integral + "." + ident("region") |
ident("region")
)))("term_desc") + "(" \
+ Optional(args, default=[''])("args") + ")"
term.setParseAction(collect_term(term_descs, lc))
rhs1 = equal + OneOrMore(term)
rhs2 = equal + zero
equation = StringStart() + OneOrMore(term) \
+ Optional(rhs1 | rhs2) + StringEnd()
## term.setDebug()
return equation
def _define_element():
VALIDATOR = pyparsing_common.identifier.setName('validator').setResultsName('validator')
ITEMS = _define_value().setName('items').setResultsName('items')
ITEMS_WRAPPER = Optional(Suppress('(') + ITEMS + Suppress(')'))
PARAMS_KEY = pyparsing_common.identifier.setName('key').setResultsName('key')
PARAMS_VALUE = _define_value().setName('value').setResultsName('value')
PARAMS_VALUE_WRAPPER = Optional(Suppress('(') + PARAMS_VALUE + Suppress(')'))
PARAMS_KEY_VALUE = Group(Suppress('.') + PARAMS_KEY + PARAMS_VALUE_WRAPPER)
PARAMS = Group(ZeroOrMore(PARAMS_KEY_VALUE)).setName('params').setResultsName('params')
return StringStart() + VALIDATOR + ITEMS_WRAPPER + PARAMS + StringEnd()
def parse(s):
path = Word(printables) ^ QuotedString('"')
option_name = Literal("exec") ^ Literal("include") ^ Literal("exclude")
option_value = path
option = Group(option_name + option_value)
options_list = Group(ZeroOrMore(option))
path_entry = Group(path + Literal("{") + \
options_list + \
Literal("}"))
path_entries_list = ZeroOrMore(path_entry)
config = StringStart() + path_entries_list + StringEnd()
entries = []
for e in config.parseString(remove_comments(s)):
opts = []
if len(e) == 4:
for o in e[2]:
opts.append(Option(o[0], sanitize(o[1])))
entries.append(Entry(sanitize(e[0]), opts))
return entries
def expr(self) -> ParserElement:
MENTION = Combine(
"[" + Optional(
SkipTo("|", failOn="]") + Suppress("|"),
default="",
) + "~" + Optional(CaselessLiteral("accountid:")) +
Word(alphanums + ":-").setResultsName("accountid") + "]", )
return ((StringStart()
| Optional(PrecededBy(White(), retreat=1), default=" ")) +
MENTION.setParseAction(self.action) +
(StringEnd() | Optional(FollowedBy(
White() | Char(punctuation, excludeChars="[") | MENTION),
default=" ")))
def expr(self) -> ParserElement:
NON_ALPHANUMS = Regex(r"\W", flags=re.UNICODE)
TOKEN = Suppress(self.TOKEN)
IGNORE = White() + TOKEN | self.get_ignore_expr()
ELEMENT = Combine(
TOKEN + (~White() & ~Char(self.TOKEN)) +
SkipTo(TOKEN, ignore=IGNORE, failOn="\n") + TOKEN +
FollowedBy(NON_ALPHANUMS | StringEnd()), )
return (StringStart()
| PrecededBy(NON_ALPHANUMS, retreat=1)) + Combine(
ELEMENT.setParseAction(self.action) +
Optional(~ELEMENT, default=" "), )
def parse(self, string):
# An integer value
integer = Word(nums)
integer.setParseAction(Integer.parse)
integer.setName("integer")
# An expression in dice notation
expression = StringStart() + operatorPrecedence(integer, [
(CaselessLiteral('d').suppress(), 2, opAssoc.LEFT, Dice.parse_binary),
(CaselessLiteral('d').suppress(), 1, opAssoc.RIGHT, Dice.parse_unary),
(Literal('/').suppress(), 2, opAssoc.LEFT, Div.parse),
(Literal('*').suppress(), 2, opAssoc.LEFT, Mul.parse),
(Literal('-').suppress(), 2, opAssoc.LEFT, Sub.parse),
(Literal('+').suppress(), 2, opAssoc.LEFT, Add.parse),
(CaselessLiteral('t').suppress(), 1, opAssoc.LEFT, Total.parse),
(CaselessLiteral('s').suppress(), 1, opAssoc.LEFT, Sort.parse),
(Literal('^').suppress(), 2, opAssoc.LEFT, Keep.parse),
(Literal('v').suppress(), 2, opAssoc.LEFT, Drop.parse),
]) + StringEnd()
expression.setName("expression")
def process_task_lists(content: str) -> str:
item = Group(CharsNotIn('\n') + (StringEnd() | '\n')).leaveWhitespace()
checkbox = oneOf(['[ ]', '[x]'])
marker = Suppress(oneOf(['+', '-', '*']) | Word(nums) + '.')
#
indent = oneOf([' ', '\t']).leaveWhitespace()
indents = Group(ZeroOrMore(indent))
#
list_item = Group(indents + marker + checkbox + item)
#
before = Suppress(StringStart() | Literal('\n\n')).leaveWhitespace()
list_ = before + OneOrMore(list_item)
#
list_.setParseAction(replace_list)
return list_.transformString(content)
def _make_key_parser():
from pyparsing import (Optional, Literal, Word, Group, Keyword,
StringStart, StringEnd, Or)
digits = '0123456789'
modifier_keywords = Word(''.join(aenea.config.MODIFIERS))
key_symbols = Or([Keyword(symbol) for symbol in aenea.config.KEYS])
pause_clause = Optional(Literal('/') + Word('.' + digits))
modifier_clause = Optional(modifier_keywords + Literal('-'))
key_hold_clause = Literal(':') + Or([Keyword(d) for d in ('up', 'down')])
keypress_clause = Group(
Group(pause_clause) + Group(Optional(Literal(':') + Word(digits))))
return (StringStart() + Group(modifier_clause) + Group(key_symbols) +
Group(key_hold_clause | keypress_clause) + Group(pause_clause) +
StringEnd())
def make_parser(payload_specification):
"""
The parser has to be payload_specification specific because it needs to know
the types of the records to build a FieldSelection.
"""
def make_selection(string, location, tokens):
variable, comparator, value = tokens
if payload_specification and variable not in payload_specification:
raise RuntimeError("%s is not a value in the folder payload "
"specification")
elif not payload_specification:
return (variable, comparator, value)
typeid = payload_specification[variable].storageType().id()
return make_fieldselection(variable, typeid, comparator, value)
from PyCool import cool
EQ = Literal("==").setParseAction(lambda t: cool.FieldSelection.EQ)
NE = Literal("!=").setParseAction(lambda t: cool.FieldSelection.NE)
GE = Literal(">=").setParseAction(lambda t: cool.FieldSelection.GE)
LE = Literal("<=").setParseAction(lambda t: cool.FieldSelection.LE)
GT = Literal(">").setParseAction(lambda t: cool.FieldSelection.GT)
LT = Literal("<").setParseAction(lambda t: cool.FieldSelection.LT)
comparator = EQ | NE | GE | LE | GT | LT
operand = (variable + comparator + value).setParseAction(make_selection)
orop = Literal("or").suppress()
andop = Literal("and").suppress()
def logic_builder(connective):
def thunk(string, location, tokens):
vec = make_selection_vector()
for token in tokens[0]:
vec.push_back(token)
return cool.CompositeSelection(connective, vec)
return thunk
expr = StringStart() + operatorPrecedence(operand, [
(andop, 2, opAssoc.LEFT, logic_builder(cool.CompositeSelection.AND)),
(orop, 2, opAssoc.LEFT, logic_builder(cool.CompositeSelection.OR)),
]) + StringEnd()
def go(selection):
return expr.parseString(selection)[0]
return go
def __init__( self ):
# define grammar
# definition of a float number
point = Literal('.')
plusOrMinus = Literal('+') | Literal('-')
number = Word(nums)
integer = Combine( Optional(plusOrMinus) + number )
floatNumber = Combine( integer + Optional( point + number ) )
# operators
plus = Literal( "+" )
minus = Literal( "-" )
mult = Literal( "*" )
div = Literal( "/" )
# parentheses are discarded
lPar = Literal( "(" ).suppress()
rPar = Literal( ")" ).suppress()
# For precedence handling
addOp = plus | minus
multOp = mult | div
logOp = Literal( "log" )
expr = Forward()
term = Forward()
atom = floatNumber | ( lPar + expr + rPar )
factor = Group(logOp + atom) | atom
term << ( Group(factor + multOp + term ) | factor )
expr << ( Group(term + addOp + expr ) | term )
self.bnf = StringStart() + expr + StringEnd()
self.ops = {
"+" : lambda a, b: a + b,
"-" : lambda a, b: a - b,
"*" : lambda a, b: a * b,
"/" : lambda a, b: a / b
}
self.funcs = {
"log" : lambda a: log10(a)
}
def expr(self) -> ParserElement:
NL = LineEnd().suppress()
SEP = (Literal("||") | Literal("|")).suppress()
ROW_BREAK = NL + SEP | NL + NL | StringEnd()
IGNORE = (Link(**self.init_kwargs).expr
| MailTo(**self.init_kwargs).expr
| Image(**self.init_kwargs).expr
| Mention(**self.init_kwargs).expr)
ROW = SEP + ZeroOrMore(
SkipTo(SEP | ROW_BREAK, ignore=IGNORE) + Optional(SEP),
stopOn=ROW_BREAK | NL + ~SEP,
)
EMPTY_LINE = Combine("\n" + White(" \t", min=0) + "\n")
return (((StringStart() + Optional("\n"))
^ Optional(EMPTY_LINE, default="\n")) +
OneOrMore(LineStart() + Group(ROW) + NL).setParseAction(
self.action) +
(StringEnd() | Optional(LineEnd(), default="\n")))
def _define_grammar(self):
g = {}
label = Literal('Contents') | Literal('Caption title') | \
Literal('Sub-caption') | Literal('Half-title') | \
Literal('Footline') | Literal('Comments') | \
Literal('Modificatons') | Literal('Errors') | \
Literal('DMF') | Literal('ADF')
copies_label = LineStart() + Literal('Copies')
all_chars = u''.join(
unichr(c) for c in xrange(65536)
if unicodedata.category(unichr(c)).startswith('L'))
section_separator = LineEnd() + FollowedBy(label | copies_label
| StringEnd())
section = SkipTo(section_separator)
library = Combine(Word(all_chars) + Literal(u'-') + Word(all_chars))
copy_separator = LineEnd() + FollowedBy(library) | \
LineEnd() + StringEnd() | StringEnd()
copy = library + SkipTo(copy_separator) + Suppress(copy_separator)
g['comments'] = Suppress('Comments') + SkipTo(section_separator)
g['code'] = StringStart() + SkipTo(LineEnd()) + Suppress(LineEnd())
g['title'] = Suppress(g['code']) + Suppress(LineEnd()) + section
g['copies'] = Suppress(copies_label) + OneOrMore(Group(copy))
return g
def build_parser(self):
self._parser = (StringStart() + self.define_operation() + StringEnd())
operators = OPERATORS_MAP.get_precedence_list(__build_unary_operator,
__build_binary_operator)
# Function call
arguments_list_call = (pOptional(delimitedList(expression)).setName(
"arguments_list_call").setParseAction(lambda r: ArgListCall(list(r))))
arguments_list_def = (pOptional(delimitedList(identifier)).setName(
"arguments_list_def").setParseAction(lambda r: ArgListDef(list(r))))
orphan_function_call_paren << (
identifier + lparen + arguments_list_call + rparen
).setName("orphan_function_call_paren").setParseAction(lambda r: FunCall(*r))
function_call_paren = ((member_access + lparen + arguments_list_call +
rparen).setName("function_call_paren").setParseAction(
lambda r: FunCall(r[0], r[1])))
function_call_no_paren = ((StringStart() + member_access +
arguments_list_call +
StringEnd()).setName("function_call_no_paren").
setParseAction(lambda r: FunCall(r[0], r[1])))
terminal = literal | function_call_paren | member_access
expression << infixNotation(terminal, operators, lpar=lparen,
rpar=rparen).setName("expression")
expression_statement = (function_call_no_paren
| expression).setName("expression_statement")
##################
# Declarations #
##################
# Variable
filterlist = OneOrMore(filter_)
or_ = Suppress(Literal('|')) + filterlist
or_.setParseAction(lambda s, l, t: pureldap.LDAPFilter_or(t))
or_.setName('or')
and_ = Suppress(Literal('&')) + filterlist
and_.setParseAction(lambda s, l, t: pureldap.LDAPFilter_and(t))
and_.setName('and')
filtercomp = and_ | or_ | not_ | item
filtercomp.setName('filtercomp')
filter_ << (Suppress(Literal('(').leaveWhitespace()) + filtercomp +
Suppress(Literal(')').leaveWhitespace()))
filter_.setName('filter')
filtercomp.leaveWhitespace()
filter_.leaveWhitespace()
toplevel = (StringStart().leaveWhitespace() + filter_ +
StringEnd().leaveWhitespace())
toplevel.leaveWhitespace()
toplevel.setName('toplevel')
def parseFilter(s):
"""
Converting source string to pureldap.LDAPFilter
Source string is converted to unicode for Python 3
as pyparsing cannot parse Python 3 byte strings with
the rules declared in this module.
"""
s = to_unicode(s) if not six.PY2 else s
try:
# An integer value
integer = Word(nums)
integer.setParseAction(Integer.parse)
integer.setName("integer")
dice_separators = RandomElement.DICE_MAP.keys()
dice_element = Or(
wrap_string(CaselessLiteral, x, suppress=False) for x in dice_separators
)
special = wrap_string(Literal, "%", suppress=False) | wrap_string(
CaselessLiteral, "f", suppress=False
)
# An expression in dice notation
expression = (
StringStart()
+ operatorPrecedence(
integer,
[
(dice_element, 2, opAssoc.LEFT, RandomElement.parse, special),
(dice_element, 1, opAssoc.RIGHT, RandomElement.parse_unary, special),
(wrap_string(CaselessLiteral, "x"), 2, opAssoc.LEFT, Explode.parse),
(wrap_string(CaselessLiteral, "x"), 1, opAssoc.LEFT, Explode.parse),
(wrap_string(CaselessLiteral, "rr"), 2, opAssoc.LEFT, ForceReroll.parse),
(wrap_string(CaselessLiteral, "rr"), 1, opAssoc.LEFT, ForceReroll.parse),
(wrap_string(CaselessLiteral, "r"), 2, opAssoc.LEFT, Reroll.parse),
(wrap_string(CaselessLiteral, "r"), 1, opAssoc.LEFT, Reroll.parse),
(wrap_string(Word, "^hH", exact=1), 2, opAssoc.LEFT, Highest.parse),
(wrap_string(Word, "^hH", exact=1), 1, opAssoc.LEFT, Highest.parse),
(wrap_string(Word, "vlL", exact=1), 2, opAssoc.LEFT, Lowest.parse),
(wrap_string(Word, "vlL", exact=1), 1, opAssoc.LEFT, Lowest.parse),
def expr(self) -> ParserElement:
return ("\n" | StringStart()) + Literal("bq. ").setParseAction(
replaceWith("> "))
def parse_query_string(query_string):
"""
Function that parse the querystring, extracting infos for limit, offset,
ordering, filters, attribute and extra projections.
:param query_string (as obtained from request.query_string)
:return: parsed values for the querykeys
"""
from pyparsing import Word, alphas, nums, alphanums, printables, \
ZeroOrMore, OneOrMore, Suppress, Optional, Literal, Group, \
QuotedString, Combine, \
StringStart as SS, StringEnd as SE, \
WordStart as WS, WordEnd as WE, \
ParseException
from pyparsing import pyparsing_common as ppc
from dateutil import parser as dtparser
from psycopg2.tz import FixedOffsetTimezone
## Define grammar
# key types
key = Word(alphas + '_', alphanums + '_')
# operators
operator = (Literal('=like=') | Literal('=ilike=') | Literal('=in=')
| Literal('=notin=') | Literal('=') | Literal('!=')
| Literal('>=') | Literal('>') | Literal('<=') | Literal('<'))
# Value types
valueNum = ppc.number
valueBool = (Literal('true')
| Literal('false')).addParseAction(lambda toks: bool(toks[0]))
valueString = QuotedString('"', escQuote='""')
valueOrderby = Combine(Optional(Word('+-', exact=1)) + key)
## DateTimeShift value. First, compose the atomic values and then combine
# them and convert them to datetime objects
# Date
valueDate = Combine(
Word(nums, exact=4) + Literal('-') + Word(nums, exact=2) +
Literal('-') + Word(nums, exact=2))
# Time
valueTime = Combine(
Literal('T') + Word(nums, exact=2) +
Optional(Literal(':') + Word(nums, exact=2)) +
Optional(Literal(':') + Word(nums, exact=2)))
# Shift
valueShift = Combine(
Word('+-', exact=1) + Word(nums, exact=2) +
Optional(Literal(':') + Word(nums, exact=2)))
# Combine atomic values
valueDateTime = Combine(
valueDate + Optional(valueTime) + Optional(valueShift) +
WE(printables.translate(None, '&')) # To us the
# word must end with '&' or end of the string
# Adding WordEnd only here is very important. This makes atomic
# values for date, time and shift not really
# usable alone individually.
)
############################################################################
# Function to convert datetime string into datetime object. The format is
# compliant with ParseAction requirements
def validate_time(s, loc, toks):
datetime_string = toks[0]
# Check the precision
precision = len(datetime_string.replace('T', ':').split(':'))
# Parse
try:
dt = dtparser.parse(datetime_string)
except ValueError:
raise RestInputValidationError("time value has wrong format. The "
"right format is "
"<date>T<time><offset>, "
"where <date> is expressed as "
"[YYYY]-[MM]-[DD], "
"<time> is expressed as [HH]:[MM]:["
"SS], "
"<offset> is expressed as +/-[HH]:["
"MM] "
"given with "
"respect to UTC")
if dt.tzinfo is not None:
tzoffset_minutes = int(
dt.tzinfo.utcoffset(None).total_seconds() / 60)
return datetime_precision(
dt.replace(tzinfo=FixedOffsetTimezone(offset=tzoffset_minutes,
name=None)), precision)
else:
return datetime_precision(
dt.replace(tzinfo=FixedOffsetTimezone(offset=0, name=None)),
precision)
########################################################################
# Convert datetime value to datetime object
valueDateTime.setParseAction(validate_time)
# More General types
value = (valueString | valueBool | valueDateTime | valueNum | valueOrderby)
# List of values (I do not check the homogeneity of the types of values,
# query builder will do it in a sense)
valueList = Group(value + OneOrMore(Suppress(',') + value) +
Optional(Suppress(',')))
# Fields
singleField = Group(key + operator + value)
listField = Group(key + (Literal('=in=') | Literal('=notin=')) + valueList)
orderbyField = Group(key + Literal('=') + valueList)
Field = (listField | orderbyField | singleField)
# Fields separator
separator = Suppress(Literal('&'))
# General query string
generalGrammar = SS() + Optional(Field) + ZeroOrMore(separator + Field) + \
Optional(separator) + SE()
## Parse the query string
try:
fields = generalGrammar.parseString(query_string)
field_dict = fields.asDict()
field_list = fields.asList()
except ParseException as e:
raise RestInputValidationError("The query string format is invalid. "
"Parser returned this massage: \"{"
"}.\" Please notice that the column "
"number "
"is counted from "
"the first character of the query "
"string.".format(e))
## return the translator instructions elaborated from the field_list
return build_translator_parameters(field_list)
type_title = 2;
s = {};
def set_label(string, loc, toks): s["label"].append(toks[1])
def set_content(string, loc, toks): s["content"] = toks[0]
def set_head(string, loc, toks): s["type"] = type_head;
def set_subhead(string, loc, toks): s["type"] = type_subhead;
def set_title(string, loc, toks): s["type"] = type_title;
Label = (Literal("[") + SkipTo(']') + "]").setParseAction(set_label);
Sentence = (Word(alphanums + " ")).setParseAction(set_content);
Subhead = (Literal("+") + ZeroOrMore(Label) + Sentence).setParseAction(set_subhead)
Headline = (Word(nums) + Literal(".") + ZeroOrMore(Label) + Sentence).setParseAction(set_head)
Title = (Literal("*") + Sentence).setParseAction(set_title)
Line = StringStart() + (Subhead | Headline | Title);
# Read a todo list, generate a schedule for the scheduler to optimize.
curr_headline = "";
items = [];
for line in open(sys.argv[1], "r"):
s.clear();
s["label"] = [];
try:
L=Line.parseString(line);
except ParseException,err:
continue;
class Parser:
def __init__( self ):
# define grammar
# definition of a float number
point = Literal('.')
plusOrMinus = Literal('+') | Literal('-')
number = Word(nums)
integer = Combine( Optional(plusOrMinus) + number )
floatNumber = Combine( integer + Optional( point + number ) )
# operators
plus = Literal( "+" )
minus = Literal( "-" )
mult = Literal( "*" )
div = Literal( "/" )
# parentheses are discarded
lPar = Literal( "(" ).suppress()
rPar = Literal( ")" ).suppress()
# For precedence handling
addOp = plus | minus
multOp = mult | div
logOp = Literal( "log" )
expr = Forward()
term = Forward()
atom = floatNumber | ( lPar + expr + rPar )
factor = Group(logOp + atom) | atom
term << ( Group(factor + multOp + term ) | factor )
expr << ( Group(term + addOp + expr ) | term )
self.bnf = StringStart() + expr + StringEnd()
self.ops = {
"+" : lambda a, b: a + b,
"-" : lambda a, b: a - b,
"*" : lambda a, b: a * b,
"/" : lambda a, b: a / b
}
self.funcs = {
"log" : lambda a: log10(a)
}
def evaluate( self, node ):
if isinstance(node, basestring):
if node in "+-*/":
return self.ops[node]
elif node in ["log"]:
return self.funcs[node]
else:
return float(node)
else:
if len(node) == 3:
operator = self.evaluate(node[1]);
return operator(self.evaluate(node[0]), self.evaluate(node[2]))
elif len(node) == 2:
function = self.evaluate(node[0]);
return function(self.evaluate(node[1]))
else:
return self.evaluate(node[0])
def parse( self, expr ):
self.exprStack = []
try:
result = self.bnf.parseString( expr )
except ParseException,err:
raise Exception, 'Parse Failure: ' + str(err)
return { "parseResult": result, "evaluation": self.evaluate( result ) }
def parse_query_string(self, query_string):
# pylint: disable=too-many-locals
"""
Function that parse the querystring, extracting infos for limit, offset,
ordering, filters, attribute and extra projections.
:param query_string (as obtained from request.query_string)
:return: parsed values for the querykeys
"""
from pyparsing import Word, alphas, nums, alphanums, printables, \
ZeroOrMore, OneOrMore, Suppress, Optional, Literal, Group, \
QuotedString, Combine, \
StringStart as SS, StringEnd as SE, \
WordEnd as WE, \
ParseException
from pyparsing import pyparsing_common as ppc
from dateutil import parser as dtparser
from psycopg2.tz import FixedOffsetTimezone
## Define grammar
# key types
key = Word(f'{alphas}_', f'{alphanums}_')
# operators
operator = (Literal('=like=') | Literal('=ilike=') | Literal('=in=')
| Literal('=notin=') | Literal('=') | Literal('!=')
| Literal('>=') | Literal('>') | Literal('<=')
| Literal('<'))
# Value types
value_num = ppc.number
value_bool = (
Literal('true')
| Literal('false')).addParseAction(lambda toks: bool(toks[0]))
value_string = QuotedString('"', escQuote='""')
value_orderby = Combine(Optional(Word('+-', exact=1)) + key)
## DateTimeShift value. First, compose the atomic values and then
# combine
# them and convert them to datetime objects
# Date
value_date = Combine(
Word(nums, exact=4) + Literal('-') + Word(nums, exact=2) +
Literal('-') + Word(nums, exact=2))
# Time
value_time = Combine(
Literal('T') + Word(nums, exact=2) +
Optional(Literal(':') + Word(nums, exact=2)) +
Optional(Literal(':') + Word(nums, exact=2)))
# Shift
value_shift = Combine(
Word('+-', exact=1) + Word(nums, exact=2) +
Optional(Literal(':') + Word(nums, exact=2)))
# Combine atomic values
value_datetime = Combine(
value_date + Optional(value_time) + Optional(value_shift) +
WE(printables.replace('&', ''))
# To us the
# word must end with '&' or end of the string
# Adding WordEnd only here is very important. This makes atomic
# values for date, time and shift not really
# usable alone individually.
)
########################################################################
def validate_time(toks):
"""
Function to convert datetime string into datetime object. The format is
compliant with ParseAction requirements
:param toks: datetime string passed in tokens
:return: datetime object
"""
datetime_string = toks[0]
# Check the precision
precision = len(datetime_string.replace('T', ':').split(':'))
# Parse
try:
dtobj = dtparser.parse(datetime_string)
except ValueError:
raise RestInputValidationError(
'time value has wrong format. The '
'right format is '
'<date>T<time><offset>, '
'where <date> is expressed as '
'[YYYY]-[MM]-[DD], '
'<time> is expressed as [HH]:[MM]:['
'SS], '
'<offset> is expressed as +/-[HH]:['
'MM] '
'given with '
'respect to UTC')
if dtobj.tzinfo is not None and dtobj.utcoffset() is not None:
tzoffset_minutes = int(dtobj.utcoffset().total_seconds() // 60)
return DatetimePrecision(
dtobj.replace(tzinfo=FixedOffsetTimezone(
offset=tzoffset_minutes, name=None)), precision)
return DatetimePrecision(
dtobj.replace(tzinfo=FixedOffsetTimezone(offset=0, name=None)),
precision)
########################################################################
# Convert datetime value to datetime object
value_datetime.setParseAction(validate_time)
# More General types
value = (value_string | value_bool | value_datetime | value_num
| value_orderby)
# List of values (I do not check the homogeneity of the types of values,
# query builder will do it somehow)
value_list = Group(value + OneOrMore(Suppress(',') + value) +
Optional(Suppress(',')))
# Fields
single_field = Group(key + operator + value)
list_field = Group(key + (Literal('=in=') | Literal('=notin=')) +
value_list)
orderby_field = Group(key + Literal('=') + value_list)
field = (list_field | orderby_field | single_field)
# Fields separator
separator = Suppress(Literal('&'))
# General query string
general_grammar = SS() + Optional(field) + ZeroOrMore(
separator + field) + \
Optional(separator) + SE()
## Parse the query string
try:
fields = general_grammar.parseString(query_string)
# JQuery adds _=timestamp a parameter to not use cached data/response.
# To handle query, remove this "_" parameter from the query string
# For more details check issue #789
# (https://github.com/aiidateam/aiida-core/issues/789) in aiida-core
field_list = [
entry for entry in fields.asList() if entry[0] != '_'
]
except ParseException as err:
raise RestInputValidationError(
'The query string format is invalid. '
"Parser returned this massage: \"{"
"}.\" Please notice that the column "
'number '
'is counted from '
'the first character of the query '
'string.'.format(err))
## return the translator instructions elaborated from the field_list
return self.build_translator_parameters(field_list)
last = this
# Set the full expression and return it
expression <<= last
return expression
# An integer value
integer = Word(nums)
integer.setParseAction(Integer.parse)
integer.setName("integer")
# An expression in dice notation
expression = StringStart() + operatorPrecedence(integer, [
(CaselessLiteral('d').suppress(), 2, opAssoc.LEFT, Dice.parse_binary),
(CaselessLiteral('d').suppress(), 1, opAssoc.RIGHT, Dice.parse_unary),
(Literal('/').suppress(), 2, opAssoc.LEFT, Div.parse),
(Literal('*').suppress(), 2, opAssoc.LEFT, Mul.parse),
(Literal('-').suppress(), 2, opAssoc.LEFT, Sub.parse),
(Literal('+').suppress(), 2, opAssoc.LEFT, Add.parse),
(Word('+-').suppress(), 1, opAssoc.RIGHT, AddEvenSubOdd.parse),
(Word('+-').suppress(), 2, opAssoc.LEFT, AddEvenSubOdd.parse),
(CaselessLiteral('t').suppress(), 1, opAssoc.LEFT, Total.parse),
(CaselessLiteral('s').suppress(), 1, opAssoc.LEFT, Sort.parse),
(Literal('^').suppress(), 2, opAssoc.LEFT, Keep.parse),
(Literal('v').suppress(), 2, opAssoc.LEFT, Drop.parse),
]) + StringEnd()
expression.setName("expression")
Optional(~SYN_KEYWORDS + Word(TOKEN)(PROP_VALUE)))
BLOCKS = (NAMED_BLOCK | ANON_BLOCK | EMPTY_BLOCK)
SECTION << Group(
Suppress(SYN_OPEN_BRACKET) + Word(TOKEN)(TYPE_SECTION) +
Suppress(SYN_CLOSE_BRACKET) + Suppress(LineEnd()) +
Group(Optional(OneOrMore(~SECTION + STATEMENT)))(PROP_BODY))
STATEMENT << (SECTION | COMMENT | RANDOM | BLOCKS | ANON_BLOCK | CONDITIONAL
| ONELINERS)
# ENDIF is to catch unmatched `endif` statements.
ROOT << (STATEMENT | Group(SYN_CLOSERS))
GRAMMAR = StringStart() + ZeroOrMore(ROOT)(PROP_BODY) + StringEnd()
# pylint: disable=too-many-branches
def structure_impl(result):
"""Recursive structure implementation."""
body = []
for statement in result[PROP_BODY]:
if TYPE_IF in statement:
body.append({
PROP_TYPE: TYPE_IF,
PROP_VALUE: statement[TYPE_IF][PROP_VALUE],
PROP_BODY: structure_impl(statement.get(TYPE_IF))
})
if TYPE_ELSEIF in statement:
for elseif in statement[TYPE_ELSEIF]:
def crn_document_setup(modular = False):
"""Parse a formal chemical reaction network.
Args:
modular <optional:bool>: Adds an additional nesting for modules within a
CRN. Use one line per module (';' separates reactions).
Format:
# A list of reactions, optionally with reaction rates:
# <- this is a comment!
B + B -> C # [k = 1]
C + A <=> D # [kf = 1, kr = 1]
<=> A [kf = 15, kr = 6]
# Note that you can write multiple reactions in one line:
A + 2C -> E [k = 13.78]; E + F <=> 2A [kf = 13, kr = 14]
Returns:
"""
# NOTE: If you want to add support for multiple modules per line, you can use
# the '|' character.
W = Word
G = Group
S = Suppress
O = Optional
C = Combine
L = Literal
def T(x, tag):
""" Return a *Tag* to distinguish (ir)reversible reactions """
def TPA(tag):
return lambda s, l, t: t.asList() + [tag]
return x.setParseAction(TPA(tag))
crn_DWC = "".join(
[x for x in ParseElementEnhance.DEFAULT_WHITE_CHARS if x != "\n"])
ParseElementEnhance.setDefaultWhitespaceChars(crn_DWC)
identifier = W(alphas, alphanums+"_")
multiplier = W(nums)
species = G(O(multiplier) + identifier)
rate = C(W(nums) + O((L('.') + W(nums)) | (L('e') + O('-') + W(nums))))
k = G(S('[') + S('k') + S('=') + rate + S(']'))
rev_k = G(S('[') + S('kf') + S('=') + rate + S(',') + \
S('kr') + S('=') + rate + S(']'))
reaction = T(G(O(delimitedList(species, "+"))) + \
S("->") + \
G(O(delimitedList(species, "+"))) + O(k), 'irreversible')
rev_reaction = T(G(O(delimitedList(species, "+"))) + \
S("<=>") + \
G(O(delimitedList(species, "+"))) + O(rev_k), 'reversible')
expr = G(reaction | rev_reaction)
if modular :
module = G(expr + ZeroOrMore(S(";") + expr))
else :
module = expr + ZeroOrMore(S(";") + expr)
formal = G(O(S(";")) + L("formals") + S(L("=") + \
L("{")) + O(delimitedList(identifier)) + S("}"))
signal = G(O(S(";")) + L("signals") + S(L("=") + \
L("{")) + O(delimitedList(identifier)) + S("}"))
fuel = G(O(S(";")) + L("fuels") + S(L("=") + \
L("{")) + O(delimitedList(identifier)) + S("}"))
addon = formal | signal | fuel
crn = OneOrMore(module + ZeroOrMore(S(LineEnd()))) + ZeroOrMore(addon + ZeroOrMore(S(LineEnd())))
document = StringStart() + ZeroOrMore(S(LineEnd())) + crn + StringEnd()
document.ignore(pythonStyleComment)
return document
operator_expression.setParseAction(action)
this <<= (operator_expression | last)
last = this
# Set the full expression and return it
expression <<= last
return expression
# An integer value
integer = Word(nums)
integer.setParseAction(Integer.parse)
integer.setName("integer")
# An expression in dice notation
expression = StringStart() + operatorPrecedence(integer, [
(CaselessLiteral('d').suppress(), 2, opAssoc.LEFT, Dice.parse_binary),
(CaselessLiteral('d').suppress(), 1, opAssoc.RIGHT, Dice.parse_unary),
(Literal('/').suppress(), 2, opAssoc.LEFT, Div.parse),
(Literal('*').suppress(), 2, opAssoc.LEFT, Mul.parse),
(Literal('-').suppress(), 2, opAssoc.LEFT, Sub.parse),
(Literal('+').suppress(), 2, opAssoc.LEFT, Add.parse),
(CaselessLiteral('t').suppress(), 1, opAssoc.LEFT, Total.parse),
(CaselessLiteral('s').suppress(), 1, opAssoc.LEFT, Sort.parse),
(Literal('^').suppress(), 2, opAssoc.LEFT, Keep.parse),
(Literal('v').suppress(), 2, opAssoc.LEFT, Drop.parse),
]) + StringEnd()
expression.setName("expression")
def __init__(self):
filename = os.path.join(paths.lib_dir(), 'bauble.glade')
self.widgets = utils.load_widgets(filename)
self.window = self.widgets.main_window
self.window.hide()
# restore the window size
geometry = prefs[self.window_geometry_pref]
if geometry is not None:
self.window.set_default_size(*geometry)
self.window.connect('delete-event', self.on_delete_event)
self.window.connect("destroy", self.on_quit)
self.window.set_title(self.title)
try:
pixbuf = gtk.gdk.pixbuf_new_from_file(bauble.default_icon)
self.window.set_icon(pixbuf)
except Exception:
logger.warning(
_('Could not load icon from %s') % bauble.default_icon)
logger.warning(traceback.format_exc())
menubar = self.create_main_menu()
self.widgets.menu_box.pack_start(menubar)
combo = self.widgets.main_comboentry
model = gtk.ListStore(str)
combo.set_model(model)
self.populate_main_entry()
main_entry = combo.child
main_entry.connect('activate', self.on_main_entry_activate)
accel_group = gtk.AccelGroup()
main_entry.add_accelerator("grab-focus", accel_group, ord('L'),
gtk.gdk.CONTROL_MASK, gtk.ACCEL_VISIBLE)
self.window.add_accel_group(accel_group)
go_button = self.widgets.go_button
go_button.connect('clicked', self.on_go_button_clicked)
query_button = self.widgets.query_button
query_button.connect('clicked', self.on_query_button_clicked)
self.set_default_view()
# add a progressbar to the status bar
# Warning: this relies on gtk.Statusbar internals and could break in
# future versions of gtk
statusbar = self.widgets.statusbar
statusbar.set_spacing(10)
statusbar.set_has_resize_grip(True)
self._cids = []
def on_statusbar_push(sb, cid, txt):
if cid not in self._cids:
self._cids.append(cid)
statusbar.connect('text-pushed', on_statusbar_push)
# remove label from frame
frame = statusbar.get_children()[0]
#frame.modify_bg(gtk.STATE_NORMAL, gtk.gdk.color_parse('#FF0000'))
label = frame.get_children()[0]
frame.remove(label)
# replace label with hbox and put label and progress bar in hbox
hbox = gtk.HBox(False, 5)
frame.add(hbox)
hbox.pack_start(label, True, True, 0)
vbox = gtk.VBox(True, 0)
hbox.pack_end(vbox, False, True, 15)
self.progressbar = gtk.ProgressBar()
vbox.pack_start(self.progressbar, False, False, 0)
self.progressbar.set_size_request(-1, 10)
vbox.show()
hbox.show()
from pyparsing import StringStart, Word, alphanums, restOfLine, \
StringEnd
cmd = StringStart() + ':' + Word(alphanums +
'-_').setResultsName('cmd')
arg = restOfLine.setResultsName('arg')
self.cmd_parser = (cmd + StringEnd()) | (cmd + '=' + arg) | arg
combo.grab_focus()
