async def apply(self, elem, **kwargs): if "element_index" in self.f_args: kwargs["element_index"] = elem.index iterable = self.f(elem.value, **kwargs) if hasattr(iterable, "__aiter__"): i = 0 async for y in iterable: await self.output_queues.put( pypeln_utils.Element(index=elem.index + (i,), value=y) ) i += 1 else: if hasattr(iterable, "__await__"): iterable = await iterable for i, y in enumerate(iterable): if hasattr(y, "__await__"): y = await y await self.output_queues.put( pypeln_utils.Element(index=elem.index + (i,), value=y) )
def consume_iterable( worker: Worker, iterable: tp.Iterable, queue: IterableQueue, loop: asyncio.AbstractEventLoop, ): try: for i, x in enumerate(iterable): if not isinstance(x, pypeln_utils.Element): x = pypeln_utils.Element(index=(i, ), value=x) if worker.is_done: return while queue.full(): if worker.is_done: return time.sleep(pypeln_utils.TIMEOUT) asyncio.run_coroutine_threadsafe(queue.put(x), loop) asyncio.run_coroutine_threadsafe(queue.worker_done(), loop) except BaseException as e: e = queue.get_pipeline_exception(e) asyncio.run_coroutine_threadsafe(queue.raise_exception(e), loop)
def apply(self, elem, **kwargs): if "element_index" in self.f_args: kwargs["element_index"] = elem.index for i, y in enumerate(self.f(elem.value, **kwargs)): elem_y = pypeln_utils.Element(index=elem.index + (i,), value=y) self.output_queues.put(elem_y)
def apply(self, worker: Worker, elem: tp.Any, **kwargs): if "element_index" in worker.f_args: kwargs["element_index"] = elem.index for i, y in enumerate(self.f(elem.value, **kwargs)): elem_y = pypeln_utils.Element(index=elem.index + (i, ), value=y) worker.stage_params.output_queues.put(elem_y)
def __call__(self, worker: Stage, **kwargs) -> tp.Iterable: if isinstance(self.iterable, pypeln_utils.BaseStage): yield from self.iterable.to_iterable(maxsize=0, return_index=True) else: for i, x in enumerate(self.iterable): if isinstance(x, pypeln_utils.Element): yield x else: yield pypeln_utils.Element(index=(i, ), value=x)
def process(self): if isinstance(self.iterable, pypeln_utils.BaseStage): yield from self.iterable.to_iterable(maxsize=0, return_index=True) else: for i, x in enumerate(self.iterable): if isinstance(x, pypeln_utils.Element): yield x else: yield pypeln_utils.Element(index=(i,), value=x)
async def apply(self, worker: Worker, elem: tp.Any, **kwargs): if "element_index" in worker.f_args: kwargs["element_index"] = elem.index ys: tp.Union[ tp.Iterable[pypeln_utils.Element], tp.AsyncIterable[pypeln_utils.Element] ] = self.f(elem.value, **kwargs) if isinstance(ys, tp.AsyncIterable): i = 0 async for y in ys: elem_y = pypeln_utils.Element(index=elem.index + (i,), value=y) await worker.stage_params.output_queues.put(elem_y) i += 1 else: for i, y in enumerate(ys): elem_y = pypeln_utils.Element(index=elem.index + (i,), value=y) await worker.stage_params.output_queues.put(elem_y)
async def process(self): i = 0 async for x in self.to_async_iterable(): if self.pipeline_namespace.error: return if isinstance(x, pypeln_utils.Element): await self.output_queues.put(x) else: await self.output_queues.put(pypeln_utils.Element(index=(i,), value=x)) i += 1
def __call__(self, worker: Worker, **kwargs): if isinstance(self.iterable, pypeln_utils.BaseStage): for x in self.iterable.to_iterable(maxsize=0, return_index=True): worker.stage_params.output_queues.put(x) else: for i, x in enumerate(self.iterable): if isinstance(x, pypeln_utils.Element): worker.stage_params.output_queues.put(x) else: worker.stage_params.output_queues.put( pypeln_utils.Element(index=(i, ), value=x))
def process(self, worker_namespace): if isinstance(self.iterable, pypeln_utils.BaseStage): for x in self.iterable.to_iterable(maxsize=0, return_index=True): if self.pipeline_namespace.error: return self.output_queues.put(x) else: for i, x in enumerate(self.iterable): if self.pipeline_namespace.error: return if isinstance(x, pypeln_utils.Element): self.output_queues.put(x) else: self.output_queues.put(pypeln_utils.Element(index=(i,), value=x))
async def __call__(self, worker: Worker, **kwargs): iterable: tp.AsyncIterable if isinstance(self.iterable, tp.AsyncIterable): iterable = self.iterable else: sync_iterable: tp.Iterable if isinstance(self.iterable, pypeln_utils.BaseStage): sync_iterable = self.iterable.to_iterable(maxsize=self.maxsize, return_index=True) else: sync_iterable = self.iterable queue = IterableQueue(maxsize=max( q.maxsize for q in worker.stage_params.output_queues)) loop = utils.get_running_loop() loop.run_in_executor( None, lambda: FromIterable.consume_iterable(worker=worker, iterable=sync_iterable, queue=queue, loop=loop), ) iterable = queue i = 0 async for x in iterable: if not isinstance(x, pypeln_utils.Element): x = pypeln_utils.Element(index=(i, ), value=x) await worker.stage_params.output_queues.put(x) i += 1
def apply(self, worker: Stage, elem: tp.Any, **kwargs) -> tp.Iterable: if "element_index" in worker.f_args: kwargs["element_index"] = elem.index for i, y in enumerate(self.f(elem.value, **kwargs)): yield pypeln_utils.Element(index=elem.index + (i,), value=y)
def apply(self, elem, **kwargs): if "element_index" in self.f_args: kwargs["element_index"] = elem.index for i, y in enumerate(self.f(elem.value, **kwargs)): yield pypeln_utils.Element(index=elem.index + (i,), value=y)