def _build_async_flow(self):
"""initialize and build the async/storey DAG"""
try:
import storey
except ImportError:
raise GraphError("storey package is not installed, use pip install storey")
def process_step(state, step, root):
if not state._is_local_function(self.context):
return
for item in state.next or []:
next_state = root[item]
if next_state.async_object:
next_step = step.to(next_state.async_object)
process_step(next_state, next_step, root)
for step in self._steps.values():
if hasattr(step, "async_object") and step._is_local_function(self.context):
if step.kind == StepKinds.queue:
skip_stream = self.context.is_mock and step.next
if step.path and not skip_stream:
stream_path = step.path
endpoint = None
if "://" in stream_path:
endpoint, stream_path = parse_v3io_path(step.path)
stream_path = stream_path.strip("/")
step._async_object = storey.StreamTarget(
storey.V3ioDriver(endpoint), stream_path
)
else:
step._async_object = storey.Map(lambda x: x)
elif not step.async_object or not hasattr(
step.async_object, "_outlets"
):
# if regular class, wrap with storey Map
step._async_object = storey.Map(
step._handler,
full_event=step.full_event,
name=step.name,
context=self.context,
)
if not step.next and hasattr(step, "responder") and step.responder:
# if responder step (return result), add Complete()
step.async_object.to(storey.Complete(full_event=True))
self._wait_for_result = True
# todo: allow source array (e.g. data->json loads..)
source = self._source or storey.SyncEmitSource()
for next_state in self._start_steps:
next_step = source.to(next_state.async_object)
process_step(next_state, next_step, self)
for step in self._steps.values():
# add error handler hooks
if (step.on_error or self.on_error) and step.async_object:
error_step = self._steps[step.on_error or self.on_error]
step.async_object.set_recovery_step(error_step.async_object)
self._controller = source.run()
def to_step(
self,
key_field=None,
time_field=None,
):
import storey
return storey.SyncEmitSource(
key_field=self.key_field or key_field,
time_field=self.time_field or time_field,
full_event=True,
)
def _init_async_objects(context, steps):
try:
import storey
except ImportError:
raise GraphError(
"storey package is not installed, use pip install storey")
wait_for_result = False
for step in steps:
if hasattr(step, "async_object") and step._is_local_function(context):
if step.kind == StepKinds.queue:
skip_stream = context.is_mock and step.next
if step.path and not skip_stream:
stream_path = step.path
endpoint = None
if "://" in stream_path:
endpoint, stream_path = parse_v3io_path(step.path)
stream_path = stream_path.strip("/")
step._async_object = storey.StreamTarget(
storey.V3ioDriver(endpoint), stream_path)
else:
step._async_object = storey.Map(lambda x: x)
elif not step.async_object or not hasattr(step.async_object,
"_outlets"):
# if regular class, wrap with storey Map
step._async_object = storey.Map(
step._handler,
full_event=step.full_event or step._call_with_event,
input_path=step.input_path,
result_path=step.result_path,
name=step.name,
context=context,
)
if not step.next and hasattr(step, "responder") and step.responder:
# if responder step (return result), add Complete()
step.async_object.to(storey.Complete(full_event=True))
wait_for_result = True
source_args = context.get_param("source_args", {})
default_source = storey.SyncEmitSource(context=context, **source_args)
return default_source, wait_for_result
def to_step(self, key_field=None, time_field=None):
import storey
return storey.SyncEmitSource()