def on_all_components_empty(producer: Producer):
all_empty = True
for step in get_project().stepist_app.get_workers_steps():
if not step.is_empty():
all_empty = False
return all_empty
def run_stepist_worker(self,
die_on_error=True,
die_when_empty=False) -> None:
"""
Run stepist app which listening streaming service and generating
jobs for pipeline functions.
It based on components which marked "as_worker" in current pipeline.
See self.get_workers_steps for more details
:param die_on_error: If True - exit on error
:param die_when_empty: If True - exit when streaming service empty
"""
steps_to_run = self.get_workers_steps()
get_project().stepist_app.run(steps_to_run,
die_on_error=die_on_error,
die_when_empty=die_when_empty)
def jobs_iterator(self, die_when_empty=False):
"""
The way how it works a bit tricky.
We manually grab data from worker engine, but we can't just return
it because stepist session is not initialized.
For this reason we manually executing step and then just return
result of this step (result of run_job)
"""
project = get_project()
while True:
job = project.stepist_app.worker_engine.receive_job(self.step)
if job is None:
if die_when_empty:
return
project.print("No jobs, waiting ... ")
time.sleep(3)
continue
yield job.get('flow_data', None)
def __call__(self, **kwargs):
if self.default_callbacks:
get_project().print("Generating jobs for: ")
for callback in self.default_callbacks:
get_project().print("-> %s" % callback.get_handler_name())
else:
get_project().print("No callbacks found, running producer ..")
if self.repeat_on_signal:
self.run_signal_repeat(**kwargs)
elif self.repeat_times:
for _ in range(self.repeat_times):
self.run(**kwargs)
else:
self.run(**kwargs)
def __init__(self,
app_name: str,
project=None,
config=None):
"""
:param app_name: App name which identify your app inside stairs project
:param project: Custom Stairs project instance
:param config: Default app config
"""
self.project = project
if self.project is None:
self.project = project_session.get_project()
self.config = config or AttrDict()
self.dbs = self.project.dbs
self.components = self.components_cls()
self.app_name = app_name
self.project.add_app(self)
signals.on_app_created(self.app_name).send_signal(self)
def run_signal_repeat(self, **user_kwargs):
repeat_count = 0
while True:
self.run(**user_kwargs)
if self.repeat_on_signal is None:
return
get_project().print("Finish %s iteration, waiting signal" %
(repeat_count + 1))
while not self.repeat_on_signal(self):
time.sleep(self.retry_sleep_time)
repeat_count += 1
if self.repeat_times and repeat_count > self.repeat_times:
get_project().print("Producer successfully repeated %s times" %
(repeat_count - 1))
break
get_project().print("Got signal `%s`, repeating .." %
self.repeat_on_signal.__name__)
def run_signal_repeat(self, **user_kwargs):
repeat_count = 0
while True:
self.run(**user_kwargs)
if self.repeat_on_signal is None:
return
get_project().print("Finish %s iteration for batch generation. "
"Run producer:run_jobs to execute producer"
% repeat_count + 1)
while not self.repeat_on_signal(self.producer):
time.sleep(self.retry_sleep_time)
repeat_count += 1
if self.repeat_times and repeat_count > self.repeat_times:
get_project().print("Producer successfully repeated %s times" % (
repeat_count - 1))
break
get_project().print("Repeating batch producer ..." )
def run_jobs(self, die_when_empty=False):
run_jobs_processor(project=get_project(),
producers_to_run=[self],
die_when_empty=die_when_empty)
def run_worker(self):
stairs_project = project_session.get_project()
worker_engine = stairs_project.stepist_app.worker_engine
worker_engine.process(self.step)