async def embedd_batch(
self,
data: List[Any],
proc_callback: Callable[[bool], None] = None
) -> List[Optional[List[float]]]:
"""
Function perform embedding of a batch of data items.
Parameters
----------
data
A list of data that must be embedded.
proc_callback
A function that is called after each item is fully processed
by either getting a successful response from the server,
getting the result from cache or skipping the item.
Returns
-------
List of float list (embeddings) for successfully embedded
items and Nones for skipped items.
Raises
------
EmbeddingCancelledException:
If cancelled attribute is set to True (default=False).
"""
requests = []
async with AsyncClient(timeout=self.timeout,
base_url=self.server_url,
proxies=get_proxies()) as client:
for p in data:
if self._cancelled:
raise EmbeddingCancelledException()
requests.append(self._send_to_server(p, client, proc_callback))
embeddings = await asyncio.gather(*requests)
self._cache.persist_cache()
assert self.num_parallel_requests == 0
return embeddings
def _embed(self, file_path):
""" Load images and compute cache keys and send requests to
an http2 server for valid ones.
"""
if self._cancelled:
raise EmbeddingCancelledException()
image = self._image_loader.load_image_or_none(file_path,
self._target_image_size)
if image is None:
return None
image = self._image_loader.preprocess_squeezenet(image)
cache_key = self._cache.md5_hash(image)
cached_im = self._cache.get_cached_result_or_none(cache_key)
if cached_im is not None:
return cached_im
embedded_image = self.embedder.predict([image])
embedded_image = embedded_image[0][0]
self._cache.add(cache_key, embedded_image)
return embedded_image
def __check_cancelled(self):
if self._cancelled:
raise EmbeddingCancelledException()