def sumByElement(tokens):
elementsList = [t[0] for t in tokens]
duplicates = len(elementsList) > len(set(elementsList))
if duplicates:
ctr = defaultdict(int)
for t in tokens:
ctr[t[0]] += t[1]
return ParseResults([ParseResults([k, v]) for k, v in ctr.items()])
def pa(s,l,t):
t = t[0]
if len(t) > initlen:
ret = ParseResults(t[:initlen])
i = initlen
while i < len(t):
ret = ParseResults([ret] + t[i:i+incr])
i += incr
return ParseResults([ret])
def nest_operand_pairs(tokens):
tokens = tokens[0]
ret = ParseResults(tokens[:3])
remaining = iter(tokens[3:])
while True:
next_pair = (next(remaining, None), next(remaining, None))
if next_pair == (None, None):
break
ret = ParseResults([ret])
ret += ParseResults(list(next_pair))
return [ret]
def _nest_operand_pairs(tokens: List[str]) -> List[_RPNItem]:
tokensf = tokens[0]
ret = ParseResults(tokensf[:3])
remaining = iter(tokensf[3:])
while True:
next_pair = (next(remaining,None), next(remaining,None))
if next_pair == (None, None):
break
ret = ParseResults([ret])
ret += ParseResults(list(next_pair))
return [ret]
def mergeJoin(s, loc, toks, flatten=False):
union = {}
flat = []
# This only works if toks are all ParseObjects so we need to use
# Group( nodeName ) to avoid getting just a string and having the
# 'join' attribute lost.
# But that's annoying, so we will collapse the list.
for t in toks:
if isinstance(t, str):
flat.append(t)
else:
flat.extend(t)
if 'join' in t:
for (k, v) in t['join'].items():
# Follow to the root of the set
# FIXME: replace with a real union-find?
while k in union:
k = union[k]
while v in union:
v = union[v]
if k != v:
union[k] = v
if flatten:
ret = ParseResults(flat)
ret['join'] = union
return ret
else:
toks['join'] = union
def process(cls, s, loc, toks):
op, rhs = toks[0]
if op == "+":
obj = PlusSign(rhs)
elif op in {"-", "\u2212"}:
obj = MinusSign(rhs)
else:
raise ValueError("Unknown prefix operator: {}".format(sign))
return ParseResults([obj])
def parse(self, data):
"""
Returns the parsed tree.
"""
content = data.strip()
if not content:
return ParseResults()
return self.script.parseString(data, parseAll=True)
def process(cls, s, loc, toks):
l = list(reversed(
toks[0])) # reverse, as append is faster than insert(0)
while len(l) > 1:
lhs = l.pop()
op = l.pop()
if op == "!":
l.append(Factorial(lhs))
else:
raise ValueError("Unknown postfix operator: {}".format(op))
return ParseResults(l)
def ingest_parse_results(self, parse_results=None):
self.parse_results = parse_results or self.parse_results or ParseResults(
)
self.initial = set(
tuple(flu) for flu in self.parse_results.get('init', self.initial))
self.state = self.initial
# FIXME: parser sometimes returns a list of lists and other times returns a list of strings (if there's only one fluent in the goal state!)
# FIXME: DRY this up and do the same thing for the initial state in case the parser hoses it up for a single fluent too!
self.goal = [
flu if isinstance(flu, basestring) else tuple(flu)
for flu in self.parse_results.get('goal', self.goal)
]
if isinstance(self.goal[0], tuple):
self.goal = set(self.goal)
else:
self.goal = set([tuple(self.goal)])
self.domain_name = iter(k for k in parse_results.keys()
if k not in ('init', 'goal')).next()
actions = dict(
((action_name,
dict(((k2,
tuple(v3 if isinstance(v3, basestring) else tuple(v3)
for v3 in v2.asList()))
for k2, v2 in action_dict.iteritems()))) for action_name,
action_dict in self.parse_results[self.domain_name].iteritems()))
for name, action in actions.iteritems():
self.actions[name] = self.actions.get(name, {})
self.actions[name]['parameters'] = tuple(
param[1:] if param.startswith('?') else param
for param in action['parameters'])
# FIXME: DRY up this repeated code for postprocessing preconditions and effects
self.actions[name]['positive_preconditions'], self.actions[name][
'negative_preconditions'] = set(), set()
polarity = 'positive'
for flu in action['precondition']:
if flu is 'not':
polarity = 'negative'
continue
self.actions[name][polarity + '_preconditions'] |= set(
[tuple(x[1:] if x.startswith('?') else x for x in flu)])
polarity = 'positive'
self.actions[name]['positive_effects'], self.actions[name][
'negative_effects'] = set(), set()
polarity = 'positive'
for flu in action['effect']:
if flu is 'not':
polarity = 'negative'
continue
self.actions[name][polarity + '_effects'] |= set(
[tuple(x[1:] if x.startswith('?') else x for x in flu)])
polarity = 'positive'
def process(cls, s, loc, toks):
name = toks[0][0]
try:
fn = FunctionList.functions[name]
argmin = FunctionList.nargs_min[name]
argmax = FunctionList.nargs_max[name]
except KeyError:
raise UnknownFunctionError(name)
args = toks[0][1:]
if not (argmin <= len(args) <= argmax):
raise WrongNumberOfArgumentsError(name, len(args), argmin, argmax)
return ParseResults([cls(fn.canonical_name, fn.fn, args, fn.args)])
def parseImpl(self, instring, loc, doActions=True):
result = self.re.match(instring, loc)
if not result or len(result.groups()) == 0 or len("".join(
result.groups())) == 0:
raise pyparsing.ParseException(instring, loc, self.errmsg, self)
loc = result.end()
d = result.groupdict()
ret = ParseResults(result.group())
if d:
for k in d:
ret[k] = d[k]
return loc, ret
def parse(self, data):
"""
Returns the parsed tree.
"""
if isinstance(data, six.binary_type):
if data[:3] == codecs.BOM_UTF8:
encoding = 'utf-8-sig'
else:
encoding = 'latin1'
content = data.decode(encoding).strip()
else:
content = data.strip()
if not content:
return ParseResults()
return self.script.parseString(content, parseAll=True)
def process(cls, s, loc, toks):
lhs = toks[0][0]
# Find signs in exponent.
signs = []
while toks[0][2] in ("+", "-"):
signs.append(toks[0].pop(2))
if signs:
signs.append(toks[0].pop())
while len(signs) > 1:
b = signs.pop()
a = signs.pop()
signs.append(
PrefixSymbol.process(None, None,
[ParseResults([a, b])])[0])
rhs = signs[0]
else:
rhs = toks[0][2]
return cls(lhs, rhs)
def process(cls, s, loc, toks):
# Sadly, those cannot be stored in the class, as the
# subclasses are not yet defined. In the interest of
# simplicity just keep them here as a local variable.
symbols = {
"+": Plus,
"-": Minus,
"\u2212": Minus,
"*": Times,
"·": Times,
"/": Divide,
"÷": Divide,
"//": IntDivide,
}
symb_set = set(symbols.keys())
# reverse, as append is faster than insert(0)
l = list(reversed(toks[0]))
while len(l) > 1:
lhs = l.pop()
op = l.pop()
# Check if operator was omitted (i.e. a multiplication
# without sign)
if op in symb_set:
rhs = l.pop()
else:
# multiplication operator was omitted.
rhs = op
op = "*"
# We will never get, e.g., +- mixed with */ due to the
# parsing. Therefore it is fine to mix them wildly here
# without error checking.
try:
cls_ = symbols[op]
except KeyError:
raise ValueError("Unknown infix operator: {}".format(op))
l.append(cls_(lhs, rhs))
return ParseResults(l)
def turn_parseresults_to_list(s, loc, toks):
return ParseResults(normalise_templates(toks[0].asList()))
def _parse_action(tokens):
"""Wrap children in a group if there are multiple"""
if len(tokens) > 1:
return ParseResults(toklist=[tokens], name=name)
return tokens
def emptyJoinDictionary(s, loc, toks):
ret = ParseResults([])
ret['join'] = {}
return ret
def parse_join(x, op="and"):
result = [op] * (len(x) * 2 - 1)
result[0::2] = x
return ParseResults(result)
def createJoinDictionary(s, loc, toks):
ret = ParseResults(toks[0])
# FIXME: raise an error if the same id appears more than once?
# This just silently ignores the extra.
ret['join'] = {t: toks[0] for t in toks[1:] if t != toks[0]}
return ret
def __init__(self, exprs):
self.exprs = ParseResults(exprs)
def pushFunc(cls, strg, loc, toks):
_, _ = loc, strg # keep pylint happy
cls.expr_stack.append(toks[1])
return ParseResults([toks[0]])
def StatementLiteral(self, lhs, comp, rhs):
return ParseResults(lhs, "lhs", False) + \
ParseResults(comp, "comp", False) + \
ParseResults([rhs], "rhs", True)
def Parse(self, query, table=None):
"""
Parses the query string and returns (sql, sql params, words).
@param table if set, search is performed on all the fields of this
specific table, ignoring all Skype-specific keywords,
only taking into account the table: keyword
{"name": "Table name": "columns[{"name", "pk_id", }, ]}
@return (SQL string, SQL parameter dict, word and phrase list)
"""
words = [] # All encountered text words and quoted phrases
keywords = collections.defaultdict(
list) # {"from": [], "chat": [], ..}
sql_params = {
} # Parameters for SQL query {"body_like0": "%word%", ..}
try:
parse_results = self._grammar.parseString(query, parseAll=True)
except Exception:
# Grammar parsing failed: do a naive parsing into keywords and words
split_words = query.split()
for word in split_words[:]:
if self.PATTERN_KEYWORD.match(word):
_, negation, key, value, _ = self.PATTERN_KEYWORD.split(
word)
key = negation + key
keywords[key.lower()].append(value)
split_words.remove(word)
try:
parse_results = ParseResults(split_words)
except NameError: # pyparsing.ParseResults not available
parse_results = split_words
result = self._makeSQL(parse_results,
words,
keywords,
sql_params,
table=table)
if table:
skip_table = False
for kw, values in keywords.items():
if ("table" == kw and table["name"].lower() not in values) \
or ("-table" == kw and table["name"].lower() in values):
skip_table = True
break # break for kw, value in keywords.items()
if skip_table:
result = ""
else:
result = "SELECT * FROM %s WHERE %s" % (table["name"], result)
for col in table["columns"]:
if col.get("pk"):
result += " ORDER BY %s ASC" % col["name"]
break # break for col in table["columns"]
else:
if "table" in keywords: del keywords["table"]
if "-table" in keywords: del keywords["-table"]
kw_sql = self._makeKeywordsSQL(keywords, sql_params)
if not table and kw_sql:
result = "%s%s" % ("%s AND " % result if result else "", kw_sql)
return result, sql_params, words
def _ast_to_dictsql(self, input_ast):
"""
"""
# Add implicit AND operator between expressions if there is no explicit
# operator specified.
ast = []
for token, lookahead in izip_longest(input_ast, input_ast[1:]):
if token.getName() == "boolean":
# only add boolean operator if it is NOT the last token
if lookahead is not None:
ast.append(token)
continue
else:
# add non-boolean token
ast.append(token)
# if next token is boolean, continue so it can be added
if lookahead is None or lookahead.getName() == "boolean":
continue
# if next token is NOT a boolean, add implicit AND
ast.append(ParseResults('and', 'boolean'))
# dictSql stack
dss = {'operator': None, 'val1': None, 'val2': None}
success = True
dse = None
for part, lookahead in izip_longest(ast, ast[1:]):
self._logger.debug("part: %s %s" % (part, part.getName()))
# handle operators joining together expressions
if part.getName() == 'boolean':
op = part[0].lower()
dss['operator'] = op
dss['interpretation'] = {
'interpretation': op,
'operator': op,
'error': False
}
continue
# string expr that we expand to dictsql expression
elif part.getName() == 'expression':
if part.operator in self.match_operators:
tmp_success, dse = self._parse_expr(part)
success = success and tmp_success
else:
tmp_success, dse = self._ast_to_dictsql(part)
success = success and tmp_success
elif part.getName() == 'nested':
tmp_success, dse = self._ast_to_dictsql(part)
success = success and tmp_success
elif part.getName() in ('ipv6_prefix', 'ipv6_address', 'word',
'tag', 'vrf_rt', 'quoted_string'):
# dict sql expression
dse = self._string_to_dictsql(part)
self._logger.debug('string part: %s => %s' % (part, dse))
else:
raise ParserError("Unhandled part in AST: %s %s" %
(part, part.getName()))
if dss['val1'] is None:
self._logger.debug('val1 not set, using dse: %s' %
unicode(dse))
dss['val1'] = dse
else:
self._logger.debug(
"val1 is set, operator is '%s', val2 = dst: %s" %
(dss['operator'], unicode(dse)))
dss['val2'] = dse
if lookahead is not None:
if dss['val1'] is not None and dss['val2'] is not None:
dss = {'operator': None, 'val1': dss, 'val2': None}
# special handling when AST is only one expression, then we overwrite
# the dss with dse
if len(ast) == 1:
dss = dse
if len(ast) == 0:
dss = self._string_to_dictsql(ParseResults('', 'word'))
# return the final composed stack of dictsql expressions
return success, dss
def update(self,
tree: ParseResults,
frame: dict,
optional: bool = True,
specified_subjects: list = None) -> ParseResults:
"""
update a tree based on a json frame
:param tree: the query tree to be updated
:param frame: the frame in json (expected output format)
:param optional: if the extra attributes in the frame is optional, default to be true
:return: return the updated tree in ParseResults
"""
if '@context' in frame:
self.context = dict(self.context, **frame['@context'])
try:
temp = tree[1]['projection'][0]
parent = temp['var'] if 'var' in temp else temp['evar']
self.prefix = self.prefix2dict(tree[0])
self.exist_triples = self.where2triples(tree[1]['where']['part'])
except KeyError as ke:
print('KeyError when parsing existing query: ', ke)
return ParseResults([])
except Exception as e:
print(e)
return ParseResults([])
triples = []
extra = []
# convert the frame to triples in the same structure with parsed query
self.frame2triple(frame, parent, triples, extra, [])
# generate the ConstructQuery CompValue
new_query = CompValue(name='ConstructQuery')
new_template = plist([ParseResults(x[-1]) for x in triples])
new_query['template'] = new_template
for k, v in tree[-1].items():
if k not in ['projection', 'modifier']:
if k == 'where' and 'part' in v:
if optional:
for x in extra:
v['part'].append(
CompValue('OptionalGraphPattern',
graph=CompValue(
'TriplesBlock',
triples=plist([
ParseResults(y)
for y in triples[x]
]))))
else:
for x in extra:
v['part'].append(
CompValue('TriplesBlock',
triples=plist([
ParseResults(y)
for y in triples[x]
])))
if specified_subjects:
spec_data = [URIRef(x) for x in specified_subjects]
v['part'].append(
CompValue('InlineData',
var=tree[-1]['projection'][0]['var'],
value=spec_data))
new_query[k] = v
if k == 'limitoffset' and 'limit' in v:
del v['limit']
return ParseResults([tree[0], new_query])
def parse_add(*args, **kwargs):
op = kwargs.get("op", "and")
x = args[0]
for y in args[1:]:
x = x + ParseResults(op) + y
return ParseResults(x)
