def _client_ingest(self):
try:
from sonic import SearchClient, IngestClient
except ImportError:
j.builder.runtimes.python.pip_package_install("sonic-client")
from sonic import SearchClient, IngestClient
if not self._cached_client_ingest:
self._cached_client_ingest = IngestClient(host=self.host, port=self.port, password=self.password)
self._cached_client_ingest.connect()
return self._cached_client_ingest
def index(snapshot_id: str, texts: List[str]):
with IngestClient(SEARCH_BACKEND_HOST_NAME, SEARCH_BACKEND_PORT,
SEARCH_BACKEND_PASSWORD) as ingestcl:
for text in texts:
chunks = (text[i:i + MAX_SONIC_TEXT_CHUNK_LENGTH] for i in range(
0,
min(len(text), MAX_SONIC_TEXT_TOTAL_LENGTH),
MAX_SONIC_TEXT_CHUNK_LENGTH,
))
for chunk in chunks:
ingestcl.push(SONIC_COLLECTION, SONIC_BUCKET, snapshot_id,
str(chunk))
def injest(files):
with IngestClient("127.0.0.1", 1491, "SecretPassword") as ingestcl:
ingestcl.flush("files", "spekter")
for file in files:
words = file_to_words(file)
for idx, word in enumerate(words):
try:
ingestcl.push("files", "spekter",
f'{clean_file(file)}:{str(idx)}', word)
except Exception as e:
print(f"Failed to push: {file}:{str(idx)} -> {word}")
print(e)
def index(snapshot_id: str, texts: List[str]):
error_count = 0
with IngestClient(SEARCH_BACKEND_HOST_NAME, SEARCH_BACKEND_PORT, SEARCH_BACKEND_PASSWORD) as ingestcl:
for text in texts:
chunks = (
text[i:i+MAX_SONIC_TEXT_CHUNK_LENGTH]
for i in range(
0,
min(len(text), MAX_SONIC_TEXT_TOTAL_LENGTH),
MAX_SONIC_TEXT_CHUNK_LENGTH,
)
)
try:
for chunk in chunks:
ingestcl.push(SONIC_COLLECTION, SONIC_BUCKET, snapshot_id, str(chunk))
except Exception as err:
print(f'[!] Sonic search backend threw an error while indexing: {err.__class__.__name__} {err}')
error_count += 1
if error_count > MAX_SONIC_ERRORS_BEFORE_ABORT:
raise
class SonicClient(JSConfigClient):
"""
Sonic server client
usage example:-
data = {
'post:1': "this is some test text hello",
'post:2': 'this is a hello world post',
'post:3': "hello how is it going?",
'post:4': "for the love of god?",
'post:5': "for the love lorde?",
}
client = j.clients.sonic.get('main', host="127.0.0.1", port=1491, password='******')
for articleid, content in data.items():
client.push("forum", "posts", articleid, content)
print(client.query("forum", "posts", "love"))
# ['post:5', 'post:4']
print(client.suggest("forum", "posts", "lo"))
# ['lorde', 'love']
"""
_SCHEMATEXT = """
@url = jumpscale.sonic.client
name* = "" (S)
host = "127.0.0.1" (S)
port = 1491 (I)
password = "" (S)
"""
def _init(self):
self._cached_client_search = None
self._cached_client_ingest = None
self.push = self._client_ingest.push
self.pop = self._client_ingest.pop
self.count = self._client_ingest.count
self.flush = self._client_ingest.flush
self.flush_collection = self._client_ingest.flush_collection
self.flush_bucket = self._client_ingest.flush_bucket
self.flush_object = self._client_ingest.flush_object
self.query = self._client_search.query
self.suggest = self._client_search.suggest
@property
def _client_search(self):
try:
from sonic import SearchClient, IngestClient
except ImportError:
j.builders.runtimes.python.pip_package_install("sonic-client")
from sonic import SearchClient, IngestClient
if not self._cached_client_search:
self._cached_client_search = SearchClient(host=self.host,
port=self.port,
password=self.password)
self._cached_client_search.connect()
return self._cached_client_search
@property
def _client_ingest(self):
try:
from sonic import SearchClient, IngestClient
except ImportError:
j.builders.runtimes.python.pip_package_install("sonic-client")
from sonic import SearchClient, IngestClient
if not self._cached_client_ingest:
self._cached_client_ingest = IngestClient(host=self.host,
port=self.port,
password=self.password)
self._cached_client_ingest.connect()
return self._cached_client_ingest
def flush(snapshot_ids: Generator[str, None, None]):
with IngestClient(SEARCH_BACKEND_HOST_NAME, SEARCH_BACKEND_PORT, SEARCH_BACKEND_PASSWORD) as ingestcl:
for id in snapshot_ids:
ingestcl.flush_object(SONIC_COLLECTION, SONIC_BUCKET, str(id))
def client_ingest(self):
if not self.cached_client_ingest:
self.cached_client_ingest = IngestClient(host=self.host, port=self.port, password=self.password)
return self.cached_client_ingest