def getchunk():
"""
Using pyparsing, create chunk reader for chunk strings.
"""
slot = pp.Word("".join([pp.alphas, "_"]), "".join([pp.alphanums, "_"]))
special_value = pp.Group(pp.oneOf([ACTRVARIABLE, "".join([ACTRNEG, ACTRVARIABLE]), ACTRNEG, VISIONGREATER, VISIONSMALLER, "".join([VISIONGREATER, ACTRVARIABLE]), "".join([VISIONSMALLER, ACTRVARIABLE])])\
+ pp.Word("".join([pp.alphanums, "_", '"', "'"])))
strvalue = pp.QuotedString('"', unquoteResults=False)
strvalue2 = pp.QuotedString("'", unquoteResults=False)
varvalue = pp.Word("".join([pp.alphanums, "_"]))
value = varvalue | special_value | strvalue | strvalue2
chunk_reader = pp.OneOrMore(pp.Group(slot + value))
return chunk_reader
def build_attribute_parser():
quoted = pp.QuotedString('"') ^ pp.QuotedString("'")
colon = pp.Literal(':').suppress()
attribute_name = pp.Word(pp.srange('[a-z]'),
pp.srange('[a-z0-9_]')).setResultsName('name')
data_type = pp.Combine(pp.Word(pp.alphas) +
pp.SkipTo("#", ignore=quoted)).setResultsName(
'type')
default = pp.Literal('=').suppress() + pp.SkipTo(
colon, ignore=quoted).setResultsName('default')
comment = pp.Literal('#').suppress() + pp.restOfLine.setResultsName(
'comment')
return attribute_name + pp.Optional(default) + colon + data_type + comment
def craft_parse(text):
"""
"""
LineComment = pyp.Combine(pyp.Literal('::') + pyp.restOfLine).suppress()
BlockComment = pyp.Combine(
pyp.Literal(':>') + pyp.SkipTo(pyp.Literal('<:')) +
pyp.Literal('<:')).suppress()
Comment = BlockComment | LineComment
BlockComment = pyp.Combine(
pyp.Literal(':<') + pyp.Combine(
pyp.NotAny(pyp.Literal(':>')) + pyp.Word(pyp.printables + ' ')) +
pyp.Literal('>:'))
Identifier = pyp.Word(pyp.alphanums + '!#$%&()*+,./;<=>?@\\^-_`{|}~')
Value = (Comment
| pyp.QuotedString('"')
| pyp.QuotedString("'")
| Identifier.addParseAction(_type_cast_value))
LBRACKET, RBRACKET, COLON = map(pyp.Suppress, '[]:')
Function = pyp.Forward()
List = pyp.Forward()
Function << pyp.Dict(
pyp.Group(Identifier + pyp.Literal(':') +
pyp.Group(LBRACKET + pyp.ZeroOrMore(Comment | Function | List
| Value) + RBRACKET)))
List << pyp.Group(LBRACKET + pyp.ZeroOrMore(Comment | Function | List
| Value) + RBRACKET)
Program = pyp.OneOrMore(Comment | Function)
# Validate for syntax error messages:
validator = SourceValidator()
Value.setParseAction(validator.validate)
List.setParseAction(validator.validate)
Identifier.addParseAction(validator.validate)
#Comment.setParseAction(validator.validate)
Function.setParseAction(validator.validate)
Program.setParseAction(validator.validate)
syntax_error = None
try:
return __walk(Program.parseString(text)[0])
except Exception as e:
syntax_error = validator.panic()
# Now raise the exception with a clean stack trace
raise syntax_error
def _make():
# Order is important - multi-char expressions need to come before narrow
# ones.
parts = []
for cls in filter_unary:
f = pp.Literal(f"~{cls.code}") + pp.WordEnd()
f.setParseAction(cls.make)
parts.append(f)
# This is a bit of a hack to simulate Word(pyparsing_unicode.printables),
# which has a horrible performance with len(pyparsing.pyparsing_unicode.printables) == 1114060
unicode_words = pp.CharsNotIn("()~'\"" + pp.ParserElement.DEFAULT_WHITE_CHARS)
unicode_words.skipWhitespace = True
regex = (
unicode_words
| pp.QuotedString('"', escChar='\\')
| pp.QuotedString("'", escChar='\\')
)
for cls in filter_rex:
f = pp.Literal(f"~{cls.code}") + pp.WordEnd() + regex.copy()
f.setParseAction(cls.make)
parts.append(f)
for cls in filter_int:
f = pp.Literal(f"~{cls.code}") + pp.WordEnd() + pp.Word(pp.nums)
f.setParseAction(cls.make)
parts.append(f)
# A naked rex is a URL rex:
f = regex.copy()
f.setParseAction(FUrl.make)
parts.append(f)
atom = pp.MatchFirst(parts)
expr = pp.infixNotation(
atom,
[(pp.Literal("!").suppress(),
1,
pp.opAssoc.RIGHT,
lambda x: FNot(*x)),
(pp.Literal("&").suppress(),
2,
pp.opAssoc.LEFT,
lambda x: FAnd(*x)),
(pp.Literal("|").suppress(),
2,
pp.opAssoc.LEFT,
lambda x: FOr(*x)),
])
expr = pp.OneOrMore(expr)
return expr.setParseAction(lambda x: FAnd(x) if len(x) != 1 else x)
def build_attribute_parser():
quoted = pp.QuotedString('"') ^ pp.QuotedString("'")
colon = pp.Literal(":").suppress()
attribute_name = pp.Word(pp.srange("[a-z]"),
pp.srange("[a-z0-9_]")).setResultsName("name")
data_type = (
pp.Combine(pp.Word(pp.alphas) + pp.SkipTo("#", ignore=quoted))
^ pp.QuotedString("<", endQuoteChar=">",
unquoteResults=False)).setResultsName("type")
default = pp.Literal("=").suppress() + pp.SkipTo(
colon, ignore=quoted).setResultsName("default")
comment = pp.Literal("#").suppress() + pp.restOfLine.setResultsName(
"comment")
return attribute_name + pp.Optional(default) + colon + data_type + comment
def get_guild_role_parser(self, guild):
"""
Create a role parser for the specified guild.
:param guild:
:return:
"""
intersect = pp.CaselessKeyword(self.INTERSECT_TOKEN)
union = pp.CaselessKeyword(self.UNION_TOKEN)
complement = pp.CaselessKeyword(self.COMPLEMENT_TOKEN)
left_paren = pp.Literal(self.LEFT_PAREN_TOKEN)
right_paren = pp.Literal(self.RIGHT_PAREN_TOKEN)
role = pp.Word(pp.alphanums) | pp.QuotedString("'", escChar="\\")
expression = pp.Forward()
term = pp.Forward()
factor = pp.Forward()
factor <<= left_paren + expression + pp.FollowedBy(
right_paren) + right_paren | complement + factor | role
term <<= factor + intersect + term | factor
expression <<= term + union + expression | term
factor.setParseAction(self.get_guild_factor_action(guild))
term.setParseAction(self.get_guild_term_action())
expression.setParseAction(self.expression_action)
role_statement = pp.StringStart() + expression + pp.StringEnd()
return role_statement
def get_parser():
"Construct and return the parser."
left = pp.Suppress("(")
right = pp.Suppress(")")
value = pp.QuotedString(quote_char='"', esc_char="\\") | pp.CharsNotIn(")")
identifier = pp.Word(pp.alphas, pp.alphanums).set_parse_action(_Identifier)
label = (pp.Keyword("label") + left + value +
right).set_parse_action(_Label)
year = (pp.Keyword("year") + left + value + right).set_parse_action(_Year)
author = (pp.Keyword("author") + left + value +
right).set_parse_action(_Author)
orcid = (pp.Keyword("orcid") + left + value +
right).set_parse_action(_Orcid)
issn = (pp.Keyword("issn") + left + value + right).set_parse_action(_Issn)
published = (pp.Keyword("published") + left + value +
right).set_parse_action(_Published)
first = (pp.Keyword("first") + left + value +
right).set_parse_action(_First)
online = (pp.Keyword("online") + left + value +
right).set_parse_action(_Online)
modified = (pp.Keyword("modified") + left + value +
right).set_parse_action(_Modified)
no_pmid = (pp.Keyword("no_pmid") + left + right).set_parse_action(_NoPmid)
no_doi = (pp.Keyword("no_doi") + left + right).set_parse_action(_NoDoi)
no_label = (pp.Keyword("no_label") + left +
right).set_parse_action(_NoLabel)
function = (label
| year
| author
| orcid
| issn
| published
| first
| online
| modified
| no_pmid
| no_doi
| no_label)
if settings["TEMPORAL_LABELS"]:
current = (pp.Keyword("active") + left +
right).set_parse_action(_Active)
active = (pp.Keyword("active") + left + value +
right).set_parse_action(_Active)
function = function | current | active
union = pp.Literal("+").set_parse_action(_Union)
symdifference = pp.Literal("^").set_parse_action(_Symdifference)
intersection = pp.Literal("#").set_parse_action(_Intersection)
difference = pp.Literal("-").set_parse_action(_Difference)
operator = union | symdifference | difference | intersection
expression = pp.Forward()
atom = function | identifier | pp.Group(left + expression + right)
expression <<= atom + (operator + atom)[...]
expression.set_parse_action(_Expression)
expression.ignore("!" + pp.rest_of_line)
return expression
def _parse(self, expr):
self.exprStack = []
parser = pp.Forward()
ffn = (fident + pp.Literal('(').suppress() +
pp.Group(pp.Optional(pp.delimitedList(parser))).setParseAction(self._pushLen) +
pp.Literal(')').suppress())
ffield = (pp.Optional(pp.Word(pp.nums) + pp.Literal(":").suppress(), default=self.article.pk) + fident
).setParseAction(lambda strg, loc, st: dict(
article_pk=st[0],
name=st[1],
))
atom = ((ffn.setResultsName('fn') |
ffield.setResultsName('field') |
pp.QuotedString('"').setResultsName('str') |
fnumber.setResultsName('float')
).setParseAction(self._push) |
pp.Group(pp.Literal('(').suppress() + parser + pp.Literal(')').suppress()))
term = atom + pp.ZeroOrMore((pp.Combine(pp.Literal("*") | pp.Literal("/")).setResultsName('op') + atom).setParseAction(self._push))
parser << term + pp.ZeroOrMore((pp.Combine(pp.Literal("+") | pp.Literal("-")).setResultsName('op') + term).setParseAction(self._push))
parser.parseString(expr, True)
def parse(s):
"""
Parses a correctly-formatted string into a sympy expression, replacing
column names in quotes w/ sympy Symbols.
Parameters
----------
s : string
detailing query. Column names should be in quotes (use \\" to escape
quotes). Otherwise, s should follow sympy conventions.
Returns
-------
(expr, symbols)
expr: sympy.Expr
symbols: dict mapping sympy.Symbol -> string (column name)
Examples
------------
>>> parse('log(cosh("Column1")) + 3')
"""
symbols = {}
def register_var(t):
name = t[0]
if name in symbols:
return symbols[name]
else:
symbols[name] = "x{}".format(len(symbols))
return symbols[name]
pp_var << pp.QuotedString('"', "\\").setParseAction(register_var)
sympy_expr = "".join(pp_expr.parseString(s))
return parse_expr(sympy_expr), symbols
def parse_expr(s):
LPAR = pyp.Literal('(').suppress()
RPAR = pyp.Literal(')').suppress()
quote = pyp.Literal('"').suppress()
sp = pyp.OneOrMore(pyp.White()).suppress()
sps = pyp.ZeroOrMore(pyp.White()).suppress()
nums = pyp.Word(pyp.srange("[0-9]"))
num = (nums + pyp.Optional(pyp.Literal('.') + nums))\
.setParseAction(lambda toks: Num(''.join(toks)))
var = (pyp.Word(pyp.alphas.lower(), pyp.srange("[a-zA-Z0-9]")) + pyp.Optional(pyp.Literal("'"))).\
setParseAction(lambda toks: Var(toks[0], toks[1] if len(toks) > 1 else None))
chars = (pyp.QuotedString('"')).setParseAction(lambda toks: Charstr(toks[0]))
boolean = (pyp.oneOf("True False true false")).setParseAction(lambda toks: Bool(toks[0]))
term = pyp.Forward()
pterm = (LPAR + sps + term + sps + RPAR).setParseAction(lambda toks: toks[0])
term << pyp.infixNotation(num | var | pterm | chars | boolean, [
(pyp.oneOf("+ -"), 2, pyp.opAssoc.LEFT, lambda ts: BinOp(ts[0][0], ts[0][1], ts[0][2])),
])
formula = pyp.Forward()
cmpop = pyp.oneOf("== < > <= >= !=")
atom = (sps + term + sps + cmpop + sps + term + sps).\
setParseAction(lambda toks: Cmp(toks[1], toks[0], toks[2]))
patom = (LPAR + sps + atom + sps + RPAR).setParseAction(lambda toks: toks[0])
formula << pyp.infixNotation(patom, [
(pyp.oneOf("&& ||"), 2, pyp.opAssoc.LEFT, lambda ts: BinCon(ts[0][0], ts[0][1], ts[0][2])),
])
res = formula.parseString(s)
r = res[0] if len(res) > 0 else None
return r
def generate_visualmin():
visualmin = p.QuotedString(quoteChar="-V/",
unquoteResults=True,
endQuoteChar="/")
visualmin.setParseAction(lambda x: ["-V", x[0]])
return visualmin
def __parse(self):
lbrace, rbrace, semi, quote = map(pyparsing.Suppress, '{};"')
ip_address = pyparsing.Combine(pyparsing.Word(pyparsing.nums) + ('.' + pyparsing.Word(pyparsing.nums)) * 3)
hex_int = pyparsing.Word(pyparsing.hexnums, exact=2)
mac_address = pyparsing.Combine(hex_int + (':' + hex_int) * 5)
hdw_type = pyparsing.Word(pyparsing.alphanums)
yyyy_mm_dd = pyparsing.Combine((pyparsing.Word(pyparsing.nums, exact=4) |
pyparsing.Word(pyparsing.nums, exact=2)) +
('/' + pyparsing.Word(pyparsing.nums, exact=2)) * 2)
hh_mm_ss = pyparsing.Combine(pyparsing.Word(pyparsing.nums, exact=2) +
(':' + pyparsing.Word(pyparsing.nums, exact=2)) * 2)
date_ref = pyparsing.oneOf(list("0123456"))("weekday") + yyyy_mm_dd("date") + hh_mm_ss("time")
def to_datetime(tokens):
tokens["datetime"] = datetime.strptime("%(date)s %(time)s" % tokens, "%Y/%m/%d %H:%M:%S")
date_ref.setParseAction(to_datetime)
starts_stmt = "starts" + date_ref + semi
ends_stmt = "ends" + (date_ref | "never") + semi
tstp_stmt = "tstp" + date_ref + semi
tsfp_stmt = "tsfp" + date_ref + semi
hdw_stmt = "hardware" + hdw_type("type") + mac_address("mac") + semi
uid_stmt = "uid" + pyparsing.QuotedString('"')("uid") + semi
binding_stmt = "binding" + pyparsing.Word(pyparsing.alphanums) + pyparsing.Word(pyparsing.alphanums) + semi
lease_statement = starts_stmt | ends_stmt | tstp_stmt | tsfp_stmt | hdw_stmt | uid_stmt | binding_stmt
lease_def = "lease" + ip_address("ipaddress") + lbrace + \
pyparsing.Dict(pyparsing.ZeroOrMore(pyparsing.Group(lease_statement))) + rbrace
with open(self.lease_file, 'r') as file:
parsed = lease_def.scanString(file.read())
return parsed
def cross_link_db_filter_parser(inputstring):
'''
example '"{ID_Score}" > 28 AND "{Sample}" ==
"%10_1%" OR ":Sample}" == "%10_2%" OR ":Sample}"
== "%10_3%" OR ":Sample}" == "%8_1%" OR ":Sample}" == "%8_2%"'
'''
import pyparsing as pp
operator = pp.Regex(">=|<=|!=|>|<|==|in").setName("operator")
value = pp.QuotedString(
'"') | pp.Regex(
r"[+-]?\d+(:?\.\d*)?(:?[eE][+-]?\d+)?")
identifier = pp.Word(pp.alphas, pp.alphanums + "_")
comparison_term = identifier | value
condition = pp.Group(comparison_term + operator + comparison_term)
expr = pp.operatorPrecedence(condition, [
("OR", 2, pp.opAssoc.LEFT, ),
("AND", 2, pp.opAssoc.LEFT, ),
])
parsedstring = str(expr.parseString(inputstring)) \
.replace("[", "(") \
.replace("]", ")") \
.replace(",", " ") \
.replace("'", " ") \
.replace("%", "'") \
.replace("{", "float(entry['") \
.replace("}", "'])") \
.replace(":", "str(entry['") \
.replace("}", "'])") \
.replace("AND", "and") \
.replace("OR", "or")
return parsedstring
def generate_range():
searchf = p.QuotedString(quoteChar="/", unquoteResults=False)
searchb = p.QuotedString(quoteChar="?", unquoteResults=False)
term = p.Word(p.nums) ^ p.oneOf("$ .") ^ searchf ^ searchb
offset = p.oneOf("+ -") + p.Word(p.nums)
offset.setParseAction(lambda x: ''.join(x))
fullterm = term("value") + p.Optional(offset("offset"), default='0')
sep = p.Literal(",").suppress()
wholebuffer = p.Literal("%")
fullrange = fullterm("a") + p.Optional(sep + fullterm)("b")
return fullrange ^ wholebuffer("all")
def DOCSTR_BLOCK(expr, resultsName=None): # pylint: disable=invalid-name
"""Block with an optional docstring followed by one of more `expr`."""
# Copied from pyparsing.indentedBlock
def checkSubIndent(s, l, t): # pylint: disable=invalid-name
curCol = pp.col(l, s) # pylint: disable=invalid-name
if curCol > _indentation_stack[-1]:
_indentation_stack.append(curCol)
else:
raise pp.ParseException(s, l, 'not a subentry')
def checkUnindent(s, l, t): # pylint: disable=invalid-name
del t
if l >= len(s):
return
curCol = pp.col(l, s) # pylint: disable=invalid-name
if not (_indentation_stack and curCol < _indentation_stack[-1]
and curCol <= _indentation_stack[-2]):
raise pp.ParseException(s, l, 'not an unindent')
_indentation_stack.pop()
INDENT = ( # pylint: disable=invalid-name
pp.Empty() +
pp.Empty().setParseAction(checkSubIndent)).setName('INDENT')
UNDENT = pp.Empty().setParseAction(checkUnindent).setName('UNINDENT') # pylint: disable=invalid-name
docstring = Group(
Tag('docstring') + pp.QuotedString('"""', multiline=True))
code = Optional(pp.indentedBlock(expr, _indentation_stack, False), [])
if resultsName:
code = code.setResultsName(resultsName)
body = INDENT + Optional(docstring)('docstring') + code + UNDENT
return S(':') + NEWLINE - body
def inner_flatten(text):
while(True): #Got to allow for for recursive includes
changed=False
def subsitute_include(string,loc, tokens):
path=tokens[0][0][1:-1]
if path in past_includes:
warn("Denied attempt to re-include "+path )
return "" #Avoid including something already included, (no infinite loops)
else:
past_includes.add(path)
changed=True;
print("* expanding", path)
directory, file = os.path.split(path)
with Context(directory):
include_text = load_include(file)
return inner_flatten(include_text)
pp_include = (pp.LineStart()
+ pp.Literal("include").suppress()
+ pp.nestedExpr(
content=pp.QuotedString('"')
).addParseAction(subsitute_include)
)
text = pp_include.transformString(text)
if not(changed):
return text
class Tokens(object):
# shared tokens
delim_chars = '[]{},'
pause = pp.FollowedBy(pp.Word(delim_chars) | pp.StringEnd())
number = (pp.pyparsing_common.number + pause)
quoted_string = pp.QuotedString('"', escChar='\\')
true = (pp.Regex(r'(True|true|yes|on)') + pause). \
setParseAction(lambda _: True)
false = (pp.Regex(r'(False|false|no|off)') + pause). \
setParseAction(lambda _: False)
unquoted_string = pp.CharsNotIn(delim_chars). \
setParseAction(lambda toks: toks[0].strip())
empty_value = pp.Empty(). \
setParseAction(lambda _: '')
# tokens for configs
identifier = pp.pyparsing_common.identifier.copy()
comma = pp.Literal(',').suppress()
assign = pp.Literal('=').suppress()
config_value = (
number | true | false | quoted_string | unquoted_string | empty_value)
key_value_pair = (identifier + assign + config_value). \
setParseAction(lambda toks: (toks[0], toks[1]))
key_value_pair_list = pp.Optional(
key_value_pair + pp.ZeroOrMore(comma + key_value_pair))
# tokens for tags
tag = quoted_string | unquoted_string | pp.Empty().suppress()
tag_list = pp.Optional(tag + pp.ZeroOrMore(comma + tag))
def from_str(s, clazz, sympy_coefficients=False):
element_parser = clazz.parser()
if sympy_coefficients:
from sympy.parsing.sympy_parser import parse_expr
coeff_s = pp.QuotedString("[", endQuoteChar="]")
coeff_s.setParseAction(lambda t: [parse_expr(t[0])])
coeff = pp.Optional(coeff_s, 1)
else:
coeff_i = pp.Suppress("[") + pp.Word(pp.nums) + pp.Suppress("]")
coeff_i.setParseAction(lambda t: [int(t[0])])
coeff_f = pp.Suppress("[") + pp.Combine(
pp.Optional(pp.Word(pp.nums)) + "." +
pp.Optional(pp.Word(pp.nums))) + pp.Suppress("]")
coeff_f.setParseAction(lambda t: [float(t[0])])
coeff = pp.Optional(coeff_i | coeff_f, 1)
if six.PY2:
minus = pp.Literal("-")
else:
#In python 3, where str is unicode, it is easy to allow the minus sign character.
#This means you can copy from a formula in a pdf
minus = pp.Literal("-") | pp.Literal(chr(0x2212))
minus.setParseAction(lambda t: ["-"])
firstTerm = pp.Optional(minus, "+") + coeff + pp.Optional(
element_parser, "")
otherTerm = (pp.Literal("+") | minus) + coeff + pp.Optional(
element_parser, "")
expn = pp.Group(firstTerm) + pp.ZeroOrMore(pp.Group(otherTerm))
exp = expn.parseString(s, True)
x = [(b if a == "+" else -b) * clazz.from_str(c) for a, b, c in exp]
out = functools.reduce(operator.add, x)
return out
def parse(cls, search=False):
"""Parse the main query text. This method will also set the
class attribute `parsed_search` to the parsed query, and it will
return it too.
:param cls: The class object, since it is a static method
:type cls: object
:param search: Search text string if a custom search string is to be
used. False if the `cls.search` class attribute is to be used.
:type search: str
:returns: Parsed query
:rtype: list
>>> print(DocMatcher.parse('hello author = einstein'))
[['hello'], ['author', '=', 'einstein']]
>>> print(DocMatcher.parse(''))
[]
>>> print(\
DocMatcher.parse(\
'"hello world whatever =" tags = \\\'hello ====\\\''))
[['hello world whatever ='], ['tags', '=', 'hello ====']]
>>> print(DocMatcher.parse('hello'))
[['hello']]
"""
import pyparsing
cls.logger.debug('Parsing search')
search = search or cls.search
papis_alphas = pyparsing.printables.replace('=', '')
papis_key = pyparsing.Word(pyparsing.alphanums + '-')
papis_value = pyparsing.QuotedString(
quoteChar='"', escChar='\\', escQuote='\\'
) ^ pyparsing.QuotedString(
quoteChar="'", escChar='\\', escQuote='\\'
) ^ papis_key
equal = pyparsing.ZeroOrMore(" ") + \
pyparsing.Literal('=') + \
pyparsing.ZeroOrMore(" ")
papis_query = pyparsing.ZeroOrMore(
pyparsing.Group(
pyparsing.ZeroOrMore(
papis_key + equal
) + papis_value
)
)
parsed = papis_query.parseString(search)
cls.logger.debug('Parsed search = %s' % parsed)
cls.parsed_search = parsed
return cls.parsed_search
def __header_pattern(self):
date_pattern = pyp.Combine(
pyp.Word(pyp.alphas) + pyp.White() + pyp.Word(pyp.nums) +
pyp.White() + pyp.Word(pyp.nums + ':'))
hostname = pyp.Word(pyp.alphanums + '-')
pid = pyp.Suppress('postgres') + pyp.QuotedString(
'[', endQuoteChar=']') + pyp.Suppress(':' + pyp.White())
return date_pattern + hostname + pid
def get_parser(implicit_bin_op='AND'):
COLON, LBRACK, RBRACK, LBRACE, RBRACE, TILDE, CARAT = map(pp.Literal, ':[]{}~^')
LPAR, RPAR = map(pp.Suppress, '()')
AND_, OR_, NOT_, TO_ = map(pp.CaselessKeyword, 'AND OR NOT TO'.split())
LTE, LT, GTE, GT, EQ = map(pp.Literal, ['<=', '<', '>=', '>', '='])
keyword = AND_ | OR_ | NOT_ | TO_
expression = pp.Forward()
valid_keyword = pp.Regex(r'[a-zA-Z_][a-zA-Z0-9_.\[\]]*')
valid_text = pp.Regex(r'([^\s\)]+)').setParseAction(lambda t: ValidText(t[0]))
quoted_string = pp.QuotedString('"').setParseAction(lambda t: QuotedString(t[0]))
rquoted_string = (pp.QuotedString('/') + pp.Optional(pp.Regex(r'[i]'))) \
.setParseAction(lambda t: RQuotedString(t[0], t[1]) if len(t) == 2 else RQuotedString(t[0], ''))
field_text_value = (quoted_string | rquoted_string | valid_text )('text_field_value').setParseAction(lambda t: TextMatcher(t[0]))
field_operate_value = ((LTE | LT | GTE | GT | EQ) + (valid_text | quoted_string))('operate_field_value').setParseAction(lambda t: Operator(t[0], t[1]))
range_text_value = (pp.Regex(r'([^\s\]\}]+)'))
incl_range_search = pp.Group(LBRACK + range_text_value + TO_ + range_text_value + RBRACK)
incl_range_search.setParseAction(lambda t: RangeMatcher(True, t[0][1], t[0][3]))
excl_range_search = pp.Group(LBRACE + range_text_value + TO_ + range_text_value + RBRACE)
excl_range_search.setParseAction(lambda t: RangeMatcher(False, t[0][1], t[0][3]))
field_range_value = (incl_range_search | excl_range_search)('range_field_value')
multiple_field_text_value = (quoted_string | rquoted_string | valid_text)
field_multiple_value = (LPAR + pp.OneOrMore(multiple_field_text_value) + RPAR)('multiple_field_value').setParseAction(lambda t: MultipleTextMatcher(t))
field_name = valid_keyword
field_value = (field_multiple_value | field_operate_value | field_range_value | field_text_value)
field_term = pp.Group(field_name('field_name') + COLON + field_value).setParseAction(lambda t: build_term_matcher(t[0][0], t[0][2]))
term = pp.Forward()
term << (field_term | pp.Group(LPAR + expression + RPAR).setParseAction(lambda t: t[0]))
not_expression = ((NOT_ | '!').setParseAction(lambda: "NOT"), 1, pp.opAssoc.RIGHT, lambda t: NotMatcher(t[0][1]))
if implicit_bin_op == 'AND':
and_expression = (pp.Optional(AND_ | '&&').setParseAction(lambda: "AND"), 2, pp.opAssoc.LEFT, lambda t: build_binary_matcher(t[0], AndMatcher))
or_expression = ((OR_ | '||').setParseAction(lambda: "OR"), 2, pp.opAssoc.LEFT, lambda t: build_binary_matcher(t[0], OrMatcher))
else:
and_expression = ((AND_ | '&&').setParseAction(lambda: "AND"), 2, pp.opAssoc.LEFT, lambda t: build_binary_matcher(t[0], AndMatcher))
or_expression = (pp.Optional(OR_ | '||').setParseAction(lambda: "OR"), 2, pp.opAssoc.LEFT, lambda t: build_binary_matcher(t[0], OrMatcher))
expression << pp.infixNotation(term, [ not_expression, and_expression, or_expression, ])
return expression
def regex_literal():
""""Defines an expression for regular expression literals.
:AST Node: `RegexLiteral`
:Example: ``/a+b+/``
"""
regex = pyparsing.QuotedString("/")
return regex.setParseAction(single_value_holder(ast.RegexLiteral))
def _make():
# Order is important - multi-char expressions need to come before narrow
# ones.
parts = []
for klass in filt_unary:
f = pp.Literal("~%s" % klass.code) + pp.WordEnd()
f.setParseAction(klass.make)
parts.append(f)
simplerex = "".join(c for c in pp.printables if c not in "()~'\"")
rex = pp.Word(simplerex) |\
pp.QuotedString("\"", escChar='\\') |\
pp.QuotedString("'", escChar='\\')
for klass in filt_rex:
f = pp.Literal("~%s" % klass.code) + pp.WordEnd() + rex.copy()
f.setParseAction(klass.make)
parts.append(f)
for klass in filt_int:
f = pp.Literal("~%s" % klass.code) + pp.WordEnd() + pp.Word(pp.nums)
f.setParseAction(klass.make)
parts.append(f)
# A naked rex is a URL rex:
f = rex.copy()
f.setParseAction(FUrl.make)
parts.append(f)
atom = pp.MatchFirst(parts)
expr = pp.operatorPrecedence(atom,
[(pp.Literal("!").suppress(),
1,
pp.opAssoc.RIGHT,
lambda x: FNot(*x)),
(pp.Literal("&").suppress(),
2,
pp.opAssoc.LEFT,
lambda x: FAnd(*x)),
(pp.Literal("|").suppress(),
2,
pp.opAssoc.LEFT,
lambda x: FOr(*x)),
])
expr = pp.OneOrMore(expr)
return expr.setParseAction(lambda x: FAnd(x) if len(x) != 1 else x)
def mix_fstring_expressions(t):
if not t.f_string_prefix:
return
fstring_arg = pp.QuotedString("{", end_quote_char="}")
fstring_arg.add_parse_action(
lambda tt: "{" + transformer.transform_string(tt[0]) + "}")
ret = t.f_string_prefix + fstring_arg.transform_string(
t.quoted_string_body)
return ret
def load_steam_folders(self):
if self.loaded_steam:
return
hkey = reg.OpenKey(reg.HKEY_CURRENT_USER, "Software\\Valve\\Steam")
steam_path = reg.QueryValueEx(hkey, "SteamPath")[0]
dl_folder = steam_path + "/steamapps/downloading"
if os.path.isdir(dl_folder):
self.directories.add(dl_folder)
# Read the steam vdf file that contains path strings to all
# game install directories.
try:
file = open(steam_path + "/steamapps/LibraryFolders.vdf").read()
except OSError:
print("Unable to open {}.".format(
steam_path + "/steamapps/LibraryFolders.vdf"))
# parse Valve's weird cfg format (its like a shitty version of JSON)
# forward declare the value of a key
value = pp.Forward()
# expression for our dict structure that looks like: ["key1", value]
key_value = pp.Group(pp.QuotedString('"') + value)
# create a parse structure for value so value looks like: c
expression = pp.Suppress(
'{') + pp.Dict(pp.ZeroOrMore(key_value)) + pp.Suppress('}')
# set our value to be a quoted string follow by the structure we defined,
# looks like this in Python:
# ["outer_key", { ["inner_key1", value], ["inner_key2", value] } ]
# we can acess the above as either a dict or array.
value <<= pp.QuotedString('"') | expression
parser = pp.Dict(key_value)
content = parser.parseString(file)
# get the last pair's key, this should be the last folder numbered folder,
# so we can use it as our max nr of folders for looping.
max_folders = int(content["LibraryFolders"][-1][0])
# loop from 1 to (incl) max folders and use it as a dictionary key to get
# the value of that key which should be a steam library folder path.
for i in range(1, max_folders + 1):
libpath = content["LibraryFolders"][str(i)]
dlpath = libpath + "\\steamapps\\downloading"
if os.path.isdir(dlpath):
self.directories.add(dlpath)
self.loaded_steam = True
def __parse_parameters(self, data):
param_pattern = pyp.delimitedList(pyp.OneOrMore(
pyp.Word('$' + pyp.nums) +
pyp.Suppress(pyp.White() + '=' + pyp.White()) +
pyp.QuotedString('\'')),
delim=",")
parsed_parameters = param_pattern.parseString(data)
iterator = iter(parsed_parameters)
parameters_dict = dict(izip(iterator, iterator))
return parameters_dict
def parse_query(query_string):
import pyparsing
logger = logging.getLogger('query_parser')
logger.debug('Parsing search')
papis_key = pyparsing.Word(pyparsing.alphanums + '-._/')
papis_value = pyparsing.QuotedString(
quoteChar='"', escChar='\\', escQuote='\\') ^ pyparsing.QuotedString(
quoteChar="'", escChar='\\', escQuote='\\') ^ papis_key
equal = (pyparsing.ZeroOrMore(" ") + pyparsing.Literal(':') +
pyparsing.ZeroOrMore(" "))
papis_query = pyparsing.ZeroOrMore(
pyparsing.Group(pyparsing.ZeroOrMore(papis_key + equal) + papis_value))
parsed = papis_query.parseString(query_string)
logger.debug('Parsed query = %s' % parsed)
return parsed
def __init__(self):
# create parsing grammer
sQStringLiteral = pyparsing.QuotedString("'")
sQStringLiteral.setParseAction(
lambda s, loc, toks: StringLiteral(s, loc, toks, False))
dQStringLiteral = pyparsing.QuotedString('"', '\\')
dQStringLiteral.setParseAction(
lambda s, loc, toks: StringLiteral(s, loc, toks, True))
stringLiteral = sQStringLiteral | dQStringLiteral
functionCall = pyparsing.Forward()
functionArg = stringLiteral | functionCall
functionCall << pyparsing.Word(pyparsing.alphas, pyparsing.alphanums+'-') + \
pyparsing.Suppress('(') + \
pyparsing.Optional(functionArg +
pyparsing.ZeroOrMore(pyparsing.Suppress(',') + functionArg)) + \
pyparsing.Suppress(')')
functionCall.setParseAction(
lambda s, loc, toks: FunctionCall(s, loc, toks))
predExpr = pyparsing.infixNotation(stringLiteral ^ functionCall, [
('!', 1, pyparsing.opAssoc.RIGHT,
lambda s, loc, toks: NotOperator(s, loc, toks)),
('<', 2, pyparsing.opAssoc.LEFT, infixBinaryOp(BinaryStrOperator)),
('<=', 2, pyparsing.opAssoc.LEFT,
infixBinaryOp(BinaryStrOperator)),
('>', 2, pyparsing.opAssoc.LEFT, infixBinaryOp(BinaryStrOperator)),
('>=', 2, pyparsing.opAssoc.LEFT,
infixBinaryOp(BinaryStrOperator)),
('==', 2, pyparsing.opAssoc.LEFT,
infixBinaryOp(BinaryStrOperator)),
('!=', 2, pyparsing.opAssoc.LEFT,
infixBinaryOp(BinaryStrOperator)),
('&&', 2, pyparsing.opAssoc.LEFT,
infixBinaryOp(BinaryBoolOperator)),
('||', 2, pyparsing.opAssoc.LEFT,
infixBinaryOp(BinaryBoolOperator))
])
self.__ifgrammer = predExpr
def __init__(self):
maybe_fuzzy = pp.Optional(pp.Literal('~')).setResultsName('fuzzy')
q_str = pp.MatchFirst([pp.QuotedString('"'), pp.QuotedString("'")])
nq_str = pp.Word(pp.alphanums + '_').setResultsName('no_quotes')
term = maybe_fuzzy + pp.MatchFirst([q_str, nq_str
]).setResultsName('term')
v_term = pp.MatchFirst([q_str, nq_str]).setResultsName('value_term')
number = pp.Word(pp.nums + '.').setResultsName('number')
string_eq = pp.MatchFirst(
[pp.Literal('!~').setResultsName('fuzzy_neq'),
pp.Literal('~')])
simple_eq = pp.MatchFirst([pp.Literal('='),
pp.Literal('!=')]).setResultsName('eq_rel')
order = pp.MatchFirst([
pp.Literal('<='),
pp.Literal('>='),
pp.Literal('<'),
pp.Literal('>')
])
num_relation = pp.MatchFirst([simple_eq, order
]).setResultsName('num_rel') + number
str_relation = pp.MatchFirst([simple_eq, string_eq]) + v_term
relation = pp.MatchFirst([num_relation,
str_relation]).setResultsName('value_test')
prop_test = pp.Group(term + pp.Optional(relation))
op_or = pp.Keyword('OR', caseless=True)
op_and = pp.Keyword('AND', caseless=True)
op_both = pp.MatchFirst([op_or, op_and])
self.parser = pp.Or([
prop_test,
pp.delimitedList(prop_test, delim=op_or).setResultsName('op_or'),
pp.delimitedList(prop_test, delim=op_and).setResultsName('op_and'),
pp.delimitedList(prop_test,
delim=op_both).setResultsName('op_both')
])
def _generate_grammar(*, debug_parser: bool = False):
pp.ParserElement.setDefaultWhitespaceChars(" \t")
EOL = pp.Optional(pp.pythonStyleComment()) + pp.LineEnd()
LC = pp.Suppress(pp.OneOrMore(EOL) + pp.White(ws=" \t", min=4))
Identifier = pp.Word(initChars=pp.alphas, bodyChars=pp.alphanums + "_-")
MultilineArgument = pp.QuotedString(
quoteChar="<<<<", endQuoteChar=">>>>", multiline=True
)
SingleQuotedArgument = pp.QuotedString(quoteChar="'", escChar="\\")
DoubleQuotedArgument = pp.QuotedString(quoteChar='"', escChar="\\")
QuotedArgument = (SingleQuotedArgument | DoubleQuotedArgument | MultilineArgument)(
"quoted"
)
SimpleArgument = pp.Word(pp.alphanums + "_-+*!$%&/()[]{}.,;:")("simple")
Argument = (QuotedArgument | SimpleArgument) + pp.Optional(LC)
KwArgument = pp.Combine(Identifier("key") + "=" + Argument)
ArgumentList = pp.Group(pp.ZeroOrMore(pp.Group(KwArgument | Argument)))
Command = (
pp.locatedExpr(Identifier)("command") + pp.Optional(LC) + ArgumentList("args")
)
Grammar = pp.ZeroOrMore(pp.Group(pp.Optional(Command) + pp.Suppress(EOL)))
if debug_parser:
for expr_name in (
"Grammar Command ArgumentList KwArgument Argument "
"SimpleArgument QuotedArgument DoubleQuotedArgument "
"SingleQuotedArgument MultilineArgument "
"Identifier LC EOL".split()
):
expr = locals()[expr_name]
expr.setName(expr_name)
expr.setDebug()
Grammar.parseWithTabs() # Keep tabs unexpanded!
return Grammar
