def _client_ingest(self): try: from sonic import SearchClient, IngestClient except ImportError: j.builder.runtimes.python.pip_package_install("sonic-client") from sonic import SearchClient, IngestClient if not self._cached_client_ingest: self._cached_client_ingest = IngestClient(host=self.host, port=self.port, password=self.password) self._cached_client_ingest.connect() return self._cached_client_ingest
def index(snapshot_id: str, texts: List[str]): with IngestClient(SEARCH_BACKEND_HOST_NAME, SEARCH_BACKEND_PORT, SEARCH_BACKEND_PASSWORD) as ingestcl: for text in texts: chunks = (text[i:i + MAX_SONIC_TEXT_CHUNK_LENGTH] for i in range( 0, min(len(text), MAX_SONIC_TEXT_TOTAL_LENGTH), MAX_SONIC_TEXT_CHUNK_LENGTH, )) for chunk in chunks: ingestcl.push(SONIC_COLLECTION, SONIC_BUCKET, snapshot_id, str(chunk))
def injest(files): with IngestClient("127.0.0.1", 1491, "SecretPassword") as ingestcl: ingestcl.flush("files", "spekter") for file in files: words = file_to_words(file) for idx, word in enumerate(words): try: ingestcl.push("files", "spekter", f'{clean_file(file)}:{str(idx)}', word) except Exception as e: print(f"Failed to push: {file}:{str(idx)} -> {word}") print(e)
def index(snapshot_id: str, texts: List[str]): error_count = 0 with IngestClient(SEARCH_BACKEND_HOST_NAME, SEARCH_BACKEND_PORT, SEARCH_BACKEND_PASSWORD) as ingestcl: for text in texts: chunks = ( text[i:i+MAX_SONIC_TEXT_CHUNK_LENGTH] for i in range( 0, min(len(text), MAX_SONIC_TEXT_TOTAL_LENGTH), MAX_SONIC_TEXT_CHUNK_LENGTH, ) ) try: for chunk in chunks: ingestcl.push(SONIC_COLLECTION, SONIC_BUCKET, snapshot_id, str(chunk)) except Exception as err: print(f'[!] Sonic search backend threw an error while indexing: {err.__class__.__name__} {err}') error_count += 1 if error_count > MAX_SONIC_ERRORS_BEFORE_ABORT: raise
class SonicClient(JSConfigClient): """ Sonic server client usage example:- data = { 'post:1': "this is some test text hello", 'post:2': 'this is a hello world post', 'post:3': "hello how is it going?", 'post:4': "for the love of god?", 'post:5': "for the love lorde?", } client = j.clients.sonic.get('main', host="127.0.0.1", port=1491, password='******') for articleid, content in data.items(): client.push("forum", "posts", articleid, content) print(client.query("forum", "posts", "love")) # ['post:5', 'post:4'] print(client.suggest("forum", "posts", "lo")) # ['lorde', 'love'] """ _SCHEMATEXT = """ @url = jumpscale.sonic.client name* = "" (S) host = "127.0.0.1" (S) port = 1491 (I) password = "" (S) """ def _init(self): self._cached_client_search = None self._cached_client_ingest = None self.push = self._client_ingest.push self.pop = self._client_ingest.pop self.count = self._client_ingest.count self.flush = self._client_ingest.flush self.flush_collection = self._client_ingest.flush_collection self.flush_bucket = self._client_ingest.flush_bucket self.flush_object = self._client_ingest.flush_object self.query = self._client_search.query self.suggest = self._client_search.suggest @property def _client_search(self): try: from sonic import SearchClient, IngestClient except ImportError: j.builders.runtimes.python.pip_package_install("sonic-client") from sonic import SearchClient, IngestClient if not self._cached_client_search: self._cached_client_search = SearchClient(host=self.host, port=self.port, password=self.password) self._cached_client_search.connect() return self._cached_client_search @property def _client_ingest(self): try: from sonic import SearchClient, IngestClient except ImportError: j.builders.runtimes.python.pip_package_install("sonic-client") from sonic import SearchClient, IngestClient if not self._cached_client_ingest: self._cached_client_ingest = IngestClient(host=self.host, port=self.port, password=self.password) self._cached_client_ingest.connect() return self._cached_client_ingest
def flush(snapshot_ids: Generator[str, None, None]): with IngestClient(SEARCH_BACKEND_HOST_NAME, SEARCH_BACKEND_PORT, SEARCH_BACKEND_PASSWORD) as ingestcl: for id in snapshot_ids: ingestcl.flush_object(SONIC_COLLECTION, SONIC_BUCKET, str(id))
def client_ingest(self): if not self.cached_client_ingest: self.cached_client_ingest = IngestClient(host=self.host, port=self.port, password=self.password) return self.cached_client_ingest