async def run(self):
worker_info = pypeln_utils.WorkerInfo(index=0)
try:
if self.on_start is not None:
on_start_kwargs = dict(worker_info=worker_info)
kwargs = self.on_start(
**{
key: value
for key, value in on_start_kwargs.items()
if key in self.on_start_args
})
else:
kwargs = {}
if kwargs is None:
kwargs = {}
if hasattr(kwargs, "__await__"):
kwargs = await kwargs
kwargs.setdefault("worker_info", worker_info)
await self.process(**{
key: value
for key, value in kwargs.items() if key in self.f_args
})
if self.on_done is not None:
kwargs.setdefault(
"stage_status",
utils.StageStatus(),
)
done_resp = self.on_done(
**{
key: value
for key, value in kwargs.items()
if key in self.on_done_args
})
if hasattr(done_resp, "__await__"):
await done_resp
await self.output_queues.done()
except BaseException as e:
self.pipeline_namespace.error = True
await self.output_queues.done()
# for stage in self.pipeline_stages:
# await stage.input_queue.done()
try:
self.pipeline_error_queue.put_nowait(
(type(e), e,
"".join(traceback.format_exception(*sys.exc_info()))))
except BaseException as e:
print(e)
def run(self, index, worker_namespace):
worker_info = pypeln_utils.WorkerInfo(index=index)
try:
if self.on_start is not None:
on_start_kwargs = dict(worker_info=worker_info)
kwargs = self.on_start(
**{
key: value
for key, value in on_start_kwargs.items()
if key in self.on_start_args
})
else:
kwargs = {}
if kwargs is None:
kwargs = {}
kwargs.setdefault("worker_info", worker_info)
self.process(
worker_namespace,
**{
key: value
for key, value in kwargs.items() if key in self.f_args
},
)
if self.on_done is not None:
with self.stage_lock:
self.stage_namespace.active_workers -= 1
kwargs.setdefault(
"stage_status",
utils.StageStatus(namespace=self.stage_namespace,
lock=self.stage_lock),
)
self.on_done(
**{
key: value
for key, value in kwargs.items()
if key in self.on_done_args
})
self.output_queues.done()
except pypeln_utils.StopThreadException:
pass
except BaseException as e:
self.signal_error(e)
finally:
worker_namespace.done = True
def run(self):
worker_info = pypeln_utils.WorkerInfo(index=0)
if self.on_start is not None:
on_start_kwargs = dict(worker_info=worker_info)
kwargs = self.on_start(
**{
key: value
for key, value in on_start_kwargs.items()
if key in self.on_start_args
}
)
else:
kwargs = {}
if kwargs is None:
kwargs = {}
kwargs.setdefault("worker_info", worker_info)
yield from self.process(
**{key: value for key, value in kwargs.items() if key in self.f_args}
)
if self.on_done is not None:
kwargs.setdefault(
"stage_status", utils.StageStatus(),
)
self.on_done(
**{
key: value
for key, value in kwargs.items()
if key in self.on_done_args
}
)