def reconstructable(target):
'''
Create a ReconstructablePipeline from a function that returns a PipelineDefinition
or a @pipeline decorated function.
'''
from dagster.core.definitions import PipelineDefinition
if not seven.is_function_or_decorator_instance_of(target,
PipelineDefinition):
raise DagsterInvariantViolationError(
'Reconstructable target should be a function or definition produced '
'by a decorated function, got {type}.'.format(type=type(target)), )
if seven.is_lambda(target):
raise DagsterInvariantViolationError(
'Reconstructable target can not be a lambda. Use a function or '
'decorated function defined at module scope instead.')
if seven.qualname_differs(target):
raise DagsterInvariantViolationError(
'Reconstructable target "{target.__name__}" has a different '
'__qualname__ "{target.__qualname__}" indicating it is not '
'defined at module scope. Use a function or decorated function '
'defined at module scope instead.'.format(target=target))
pointer = FileCodePointer(
python_file=get_python_file_from_previous_stack_frame(),
fn_name=target.__name__,
)
return bootstrap_standalone_recon_pipeline(pointer)
def reconstructable(target):
"""
Create a ReconstructablePipeline from a function that returns a PipelineDefinition
or a @pipeline decorated function.
"""
from dagster.core.definitions import PipelineDefinition
if not seven.is_function_or_decorator_instance_of(target,
PipelineDefinition):
raise DagsterInvariantViolationError(
"Reconstructable target should be a function or definition produced "
"by a decorated function, got {type}.".format(type=type(target)), )
if seven.is_lambda(target):
raise DagsterInvariantViolationError(
"Reconstructable target can not be a lambda. Use a function or "
"decorated function defined at module scope instead.")
if seven.qualname_differs(target):
raise DagsterInvariantViolationError(
'Reconstructable target "{target.__name__}" has a different '
'__qualname__ "{target.__qualname__}" indicating it is not '
"defined at module scope. Use a function or decorated function "
"defined at module scope instead.".format(target=target))
python_file = get_python_file_from_previous_stack_frame()
if python_file.endswith("<stdin>"):
raise DagsterInvariantViolationError(
"reconstructable() can not reconstruct pipelines from <stdin>, unable to target file {}. "
.format(python_file))
pointer = FileCodePointer(python_file=python_file,
fn_name=target.__name__,
working_directory=os.getcwd())
return bootstrap_standalone_recon_pipeline(pointer)
def reconstructable(target):
"""
Create a ReconstructablePipeline from a function that returns a PipelineDefinition, or a
function decorated with :py:func:`@pipeline <dagster.pipeline>`
When your pipeline must cross process boundaries, e.g., for execution on multiple nodes or
in different systems (like dagstermill), Dagster must know how to reconstruct the pipeline
on the other side of the process boundary.
This function implements a very conservative strategy for reconstructing pipelines, so that
its behavior is easy to predict, but as a consequence it is not able to reconstruct certain
kinds of pipelines, such as those defined by lambdas, in nested scopes (e.g., dynamically
within a method call), or in interactive environments such as the Python REPL or Jupyter
notebooks.
If you need to reconstruct pipelines constructed in these ways, you should use
:py:func:`build_reconstructable_pipeline` instead, which allows you to specify your own
strategy for reconstructing a pipeline.
Examples:
.. code-block:: python
from dagster import PipelineDefinition, pipeline, recontructable
@pipeline
def foo_pipeline():
...
reconstructable_foo_pipeline = reconstructable(foo_pipeline)
def make_bar_pipeline():
return PipelineDefinition(...)
reconstructable_bar_pipeline = reconstructable(bar_pipeline)
"""
from dagster.core.definitions import PipelineDefinition
if not seven.is_function_or_decorator_instance_of(target,
PipelineDefinition):
raise DagsterInvariantViolationError(
"Reconstructable target should be a function or definition produced "
"by a decorated function, got {type}.".format(type=type(target)), )
if seven.is_lambda(target):
raise DagsterInvariantViolationError(
"Reconstructable target can not be a lambda. Use a function or "
"decorated function defined at module scope instead, or use "
"build_reconstructable_pipeline.")
if seven.qualname_differs(target):
raise DagsterInvariantViolationError(
'Reconstructable target "{target.__name__}" has a different '
'__qualname__ "{target.__qualname__}" indicating it is not '
"defined at module scope. Use a function or decorated function "
"defined at module scope instead, or use build_reconstructable_pipeline."
.format(target=target))
python_file = get_python_file_from_previous_stack_frame()
if python_file.endswith("<stdin>"):
raise DagsterInvariantViolationError(
"reconstructable() can not reconstruct pipelines from <stdin>, unable to "
"target file {}. Use a pipeline defined in a module or file instead, or "
"use build_reconstructable_pipeline.".format(python_file))
pointer = FileCodePointer(python_file=python_file,
fn_name=target.__name__,
working_directory=os.getcwd())
# ipython:
# Exception: Can not import module <ipython-input-3-70f55f9e97d2> from path /Users/max/Desktop/richard_brady_repro/<ipython-input-3-70f55f9e97d2>, unable to load spec.
# Exception: Can not import module from path /private/var/folders/zc/zyv5jx615157j4mypwcx_kxr0000gn/T/b3edec1e-b4c5-4ea4-a4ae-24a01e566aba/, unable to load spec.
return bootstrap_standalone_recon_pipeline(pointer)
