async def apply(self, elem, **kwargs):
if "element_index" in self.f_args:
kwargs["element_index"] = elem.index
iterable = self.f(elem.value, **kwargs)
if hasattr(iterable, "__aiter__"):
i = 0
async for y in iterable:
await self.output_queues.put(
pypeln_utils.Element(index=elem.index + (i,), value=y)
)
i += 1
else:
if hasattr(iterable, "__await__"):
iterable = await iterable
for i, y in enumerate(iterable):
if hasattr(y, "__await__"):
y = await y
await self.output_queues.put(
pypeln_utils.Element(index=elem.index + (i,), value=y)
)
def consume_iterable(
worker: Worker,
iterable: tp.Iterable,
queue: IterableQueue,
loop: asyncio.AbstractEventLoop,
):
try:
for i, x in enumerate(iterable):
if not isinstance(x, pypeln_utils.Element):
x = pypeln_utils.Element(index=(i, ), value=x)
if worker.is_done:
return
while queue.full():
if worker.is_done:
return
time.sleep(pypeln_utils.TIMEOUT)
asyncio.run_coroutine_threadsafe(queue.put(x), loop)
asyncio.run_coroutine_threadsafe(queue.worker_done(), loop)
except BaseException as e:
e = queue.get_pipeline_exception(e)
asyncio.run_coroutine_threadsafe(queue.raise_exception(e), loop)
def apply(self, elem, **kwargs):
if "element_index" in self.f_args:
kwargs["element_index"] = elem.index
for i, y in enumerate(self.f(elem.value, **kwargs)):
elem_y = pypeln_utils.Element(index=elem.index + (i,), value=y)
self.output_queues.put(elem_y)
def apply(self, worker: Worker, elem: tp.Any, **kwargs):
if "element_index" in worker.f_args:
kwargs["element_index"] = elem.index
for i, y in enumerate(self.f(elem.value, **kwargs)):
elem_y = pypeln_utils.Element(index=elem.index + (i, ), value=y)
worker.stage_params.output_queues.put(elem_y)
def __call__(self, worker: Stage, **kwargs) -> tp.Iterable:
if isinstance(self.iterable, pypeln_utils.BaseStage):
yield from self.iterable.to_iterable(maxsize=0, return_index=True)
else:
for i, x in enumerate(self.iterable):
if isinstance(x, pypeln_utils.Element):
yield x
else:
yield pypeln_utils.Element(index=(i, ), value=x)
def process(self):
if isinstance(self.iterable, pypeln_utils.BaseStage):
yield from self.iterable.to_iterable(maxsize=0, return_index=True)
else:
for i, x in enumerate(self.iterable):
if isinstance(x, pypeln_utils.Element):
yield x
else:
yield pypeln_utils.Element(index=(i,), value=x)
async def apply(self, worker: Worker, elem: tp.Any, **kwargs):
if "element_index" in worker.f_args:
kwargs["element_index"] = elem.index
ys: tp.Union[
tp.Iterable[pypeln_utils.Element], tp.AsyncIterable[pypeln_utils.Element]
] = self.f(elem.value, **kwargs)
if isinstance(ys, tp.AsyncIterable):
i = 0
async for y in ys:
elem_y = pypeln_utils.Element(index=elem.index + (i,), value=y)
await worker.stage_params.output_queues.put(elem_y)
i += 1
else:
for i, y in enumerate(ys):
elem_y = pypeln_utils.Element(index=elem.index + (i,), value=y)
await worker.stage_params.output_queues.put(elem_y)
async def process(self):
i = 0
async for x in self.to_async_iterable():
if self.pipeline_namespace.error:
return
if isinstance(x, pypeln_utils.Element):
await self.output_queues.put(x)
else:
await self.output_queues.put(pypeln_utils.Element(index=(i,), value=x))
i += 1
def __call__(self, worker: Worker, **kwargs):
if isinstance(self.iterable, pypeln_utils.BaseStage):
for x in self.iterable.to_iterable(maxsize=0, return_index=True):
worker.stage_params.output_queues.put(x)
else:
for i, x in enumerate(self.iterable):
if isinstance(x, pypeln_utils.Element):
worker.stage_params.output_queues.put(x)
else:
worker.stage_params.output_queues.put(
pypeln_utils.Element(index=(i, ), value=x))
def process(self, worker_namespace):
if isinstance(self.iterable, pypeln_utils.BaseStage):
for x in self.iterable.to_iterable(maxsize=0, return_index=True):
if self.pipeline_namespace.error:
return
self.output_queues.put(x)
else:
for i, x in enumerate(self.iterable):
if self.pipeline_namespace.error:
return
if isinstance(x, pypeln_utils.Element):
self.output_queues.put(x)
else:
self.output_queues.put(pypeln_utils.Element(index=(i,), value=x))
async def __call__(self, worker: Worker, **kwargs):
iterable: tp.AsyncIterable
if isinstance(self.iterable, tp.AsyncIterable):
iterable = self.iterable
else:
sync_iterable: tp.Iterable
if isinstance(self.iterable, pypeln_utils.BaseStage):
sync_iterable = self.iterable.to_iterable(maxsize=self.maxsize,
return_index=True)
else:
sync_iterable = self.iterable
queue = IterableQueue(maxsize=max(
q.maxsize for q in worker.stage_params.output_queues))
loop = utils.get_running_loop()
loop.run_in_executor(
None,
lambda: FromIterable.consume_iterable(worker=worker,
iterable=sync_iterable,
queue=queue,
loop=loop),
)
iterable = queue
i = 0
async for x in iterable:
if not isinstance(x, pypeln_utils.Element):
x = pypeln_utils.Element(index=(i, ), value=x)
await worker.stage_params.output_queues.put(x)
i += 1
def apply(self, worker: Stage, elem: tp.Any, **kwargs) -> tp.Iterable:
if "element_index" in worker.f_args:
kwargs["element_index"] = elem.index
for i, y in enumerate(self.f(elem.value, **kwargs)):
yield pypeln_utils.Element(index=elem.index + (i,), value=y)
def apply(self, elem, **kwargs):
if "element_index" in self.f_args:
kwargs["element_index"] = elem.index
for i, y in enumerate(self.f(elem.value, **kwargs)):
yield pypeln_utils.Element(index=elem.index + (i,), value=y)
