def test_lucene_resolution_simple(self):
tree = (UnknownOperation(Word("a"), Word("b"),
UnknownOperation(Word("c"), Word("d"))))
expected = (AndOperation(Word("a"), Word("b"),
AndOperation(Word("c"), Word("d"))))
resolver = UnknownOperationResolver(resolve_to=None)
self.assertEqual(resolver(tree), expected)
def test_and_resolution(self):
tree = (UnknownOperation(Word("a"), Word("b"),
OrOperation(Word("c"), Word("d"))))
expected = (AndOperation(Word("a"), Word("b"),
OrOperation(Word("c"), Word("d"))))
resolver = UnknownOperationResolver(resolve_to=AndOperation)
self.assertEqual(resolver(tree), expected)
def _parse_query(self, query: str, ambiguous_action: bool) -> None:
"""
Parse the query, replace any ambiguous (unknown) parts with the correct operation
and check for unsupported operations.
Args:
query: A Lucene style query.
ambiguous_action: The action to use for ambiguous queries, for example "field1:value1 field2:value2" (default: "AND")
"""
if query is None or query.strip() == "":
raise ValueError("Need a valid query")
try:
self._tree = parser.parse(query)
except ParseError as exc:
raise QueryException("Could not parse the query, error: {}".format(
str(exc)))
# Replace any UnknownOperation with the chosen action
if ambiguous_action != "Exception":
operation = self.ambiguous_actions[ambiguous_action]
resolver = UnknownOperationResolver(resolve_to=operation)
self._tree = resolver(self._tree)
# Raise a QueryException if the user has passed unsupported search terms
self._check_tree(self._tree)
def test_head_tail_pos(self):
tree = parser.parse("\ra\nb (c\t (d e f)) ")
resolver = UnknownOperationResolver(resolve_to=None)
transformed = resolver(tree)
self.assertEqual(str(transformed),
"\ra\nAND b AND (c\t AND (d AND e AND f)) ")
self.assertEqual(transformed.pos, tree.pos)
self.assertEqual(transformed.size, tree.size)
and_op, orig_op = transformed.children[2].children[0], tree.children[
2].children[0]
self.assertEqual(type(and_op), AndOperation)
self.assertEqual(and_op.pos, orig_op.pos)
self.assertEqual(and_op.size, orig_op.size)
and_op, orig_op = and_op.children[1].children[0], orig_op.children[
1].children[0]
self.assertEqual(type(and_op), AndOperation)
self.assertEqual(and_op.pos, orig_op.pos)
self.assertEqual(and_op.size, orig_op.size)
resolver = UnknownOperationResolver(resolve_to=OrOperation)
transformed = resolver(tree)
self.assertEqual(str(transformed),
"\ra\nOR b OR (c\t OR (d OR e OR f)) ")
def test_lucene_resolution_last_op_with_group(self):
tree = (OrOperation(
Word("a"), Word("b"),
Group(
AndOperation(Word("c"), UnknownOperation(Word("d"),
Word("e")))),
UnknownOperation(Word("f"), Word("g")),
Group(UnknownOperation(Word("h"), Word("i")))))
expected = (OrOperation(
Word("a"), Word("b"),
Group(AndOperation(Word("c"), AndOperation(Word("d"), Word("e")))),
OrOperation(Word("f"), Word("g")),
Group(AndOperation(Word("h"), Word("i")))))
resolver = UnknownOperationResolver(resolve_to=None)
self.assertEqual(resolver(tree), expected)
def test_lucene_resolution_last_op(self):
tree = (OrOperation(
Word("a"), Word("b"), UnknownOperation(Word("c"), Word("d")),
AndOperation(Word("e"), UnknownOperation(Word("f"), Word("g"))),
UnknownOperation(Word("i"), Word("j")),
OrOperation(Word("k"), UnknownOperation(Word("l"), Word("m"))),
UnknownOperation(Word("n"), Word("o"))))
expected = (OrOperation(
Word("a"), Word("b"), OrOperation(Word("c"), Word("d")),
AndOperation(Word("e"), AndOperation(Word("f"), Word("g"))),
AndOperation(Word("i"), Word("j")),
OrOperation(Word("k"), OrOperation(Word("l"), Word("m"))),
OrOperation(Word("n"), Word("o"))))
resolver = UnknownOperationResolver(resolve_to=None)
self.assertEqual(resolver(tree), expected)
def test_resolve_to_verification(self):
with self.assertRaises(ValueError):
UnknownOperationResolver(resolve_to=object())
from .models import UserAuths
from .models import Auth
from .WritableUtils import *
from .rangebuilder import *
from luqum.parser import lexer, parser, ParseError
from luqum.pretty import prettify
from luqum.utils import UnknownOperationResolver, LuceneTreeVisitorV2
from luqum.exceptions import OrAndAndOnSameLevel
from luqum.tree import OrOperation, AndOperation, UnknownOperation
from luqum.tree import Word # noqa: F401
import asyncio
from collections import deque
import queue
import concurrent.futures
resolver = UnknownOperationResolver()
import pysharkbite
class LookupInformation(object):
def __init__(self,lookupTable,auths, tableOps):
self._lookupTable=lookupTable
self._auths = auths
self._tableOps = tableOps
def getLookupTable(self):
return self._lookupTable
def getAuths(self):
import contextlib
import io
import logging
import sqlite3
import string
from luqum.parser import parser
from luqum.tree import AndOperation
from luqum.utils import LuceneTreeTransformer, UnknownOperationResolver
log = logging.getLogger(__name__)
_SAFE_QUERY_CHARS = set(
':()"'
) # "-" is not processed to NOT by luqum. ":" is required by luqum for "path:/foo". "()" are used by sqlite3 FTS5. """ is used for quoting.
_UNKNOWN_OP_RESOLVER = UnknownOperationResolver(AndOperation)
_UNSAFE_QUERY_CHARS = set(string.punctuation) - _SAFE_QUERY_CHARS
class _SearchFieldRemover(LuceneTreeTransformer):
def visit_search_field(self, node, parents): # pylint: disable=unused-argument,no-self-use
return None
class SqliteFTS5Matcher:
"""Validate whether a given GitHub code search query matches against the specified text using SQLite FTS5."""
def __init__(self, query: str) -> None:
self._query = self._convert_github_query_to_sqlite_fts5_query(query)
if query != self._query:
log.info(
f"Adjusted the query {query!r} for local validation to {self._query!r}."