def __call__(self, fn: Callable[..., Any]) -> PipelineDefinition:
check.callable_param(fn, "fn")
if not self.name:
self.name = fn.__name__
from dagster.core.definitions.decorators.composite_solid import (
do_composition,
get_validated_config_mapping,
)
config_mapping = get_validated_config_mapping(
self.name,
self.config_schema,
self.config_fn,
decorator_name="pipeline")
(
input_mappings,
output_mappings,
dependencies,
solid_defs,
config_mapping,
positional_inputs,
) = do_composition(
"@pipeline",
self.name,
fn,
self.input_defs,
self.output_defs,
config_mapping,
ignore_output_from_composition_fn=not self.did_pass_outputs,
)
pipeline_def = PipelineDefinition(
mode_defs=self.mode_definitions,
preset_defs=self.preset_definitions,
graph_def=GraphDefinition(
name=self.name,
description=None, # put desc on the pipeline
dependencies=dependencies,
node_defs=solid_defs,
input_mappings=input_mappings,
output_mappings=output_mappings,
config=config_mapping,
positional_inputs=positional_inputs,
),
tags=self.tags,
description=self.description
or format_docstring_for_description(fn),
hook_defs=self.hook_defs,
solid_retry_policy=self.solid_retry_policy,
version_strategy=self.version_strategy,
)
update_wrapper(pipeline_def, fn)
return pipeline_def
def __call__(self, fn: Callable[..., Any]) -> GraphDefinition:
check.callable_param(fn, "fn")
if not self.name:
self.name = fn.__name__
if self.ins is not None:
input_defs = [inp.to_definition(name) for name, inp in self.ins.items()]
else:
input_defs = check.opt_list_param(
self.input_defs, "input_defs", of_type=InputDefinition
)
if self.out is None:
output_defs = self.output_defs
elif isinstance(self.out, GraphOut):
output_defs = [self.out.to_definition(name=None)]
else:
check.dict_param(self.out, "out", key_type=str, value_type=GraphOut)
output_defs = [out.to_definition(name=name) for name, out in self.out.items()]
from dagster.core.definitions.decorators.composite_solid import do_composition
(
input_mappings,
output_mappings,
dependencies,
solid_defs,
config_mapping,
positional_inputs,
) = do_composition(
decorator_name="@graph",
graph_name=self.name,
fn=fn,
provided_input_defs=input_defs,
provided_output_defs=output_defs,
ignore_output_from_composition_fn=False,
config_mapping=self.config_mapping,
)
graph_def = GraphDefinition(
name=self.name,
dependencies=dependencies,
node_defs=solid_defs,
description=self.description or format_docstring_for_description(fn),
input_mappings=input_mappings,
output_mappings=output_mappings,
config=config_mapping,
positional_inputs=positional_inputs,
tags=self.tags,
)
update_wrapper(graph_def, fn)
return graph_def
def __call__(self, fn: Callable[..., Any]) -> JobDefinition:
check.callable_param(fn, "fn")
if not self.name:
self.name = fn.__name__
from dagster.core.definitions.decorators.composite_solid import do_composition
(
input_mappings,
output_mappings,
dependencies,
solid_defs,
config_mapping,
positional_inputs,
) = do_composition(
decorator_name="@job",
graph_name=self.name,
fn=fn,
provided_input_defs=[],
provided_output_defs=[],
ignore_output_from_composition_fn=False,
config_mapping=None,
)
graph_def = GraphDefinition(
name=self.name,
dependencies=dependencies,
node_defs=solid_defs,
description=self.description
or format_docstring_for_description(fn),
input_mappings=input_mappings,
output_mappings=output_mappings,
config=config_mapping,
positional_inputs=positional_inputs,
tags=self.tags,
)
job_def = graph_def.to_job(
description=self.description
or format_docstring_for_description(fn),
resource_defs=self.resource_defs,
config=self.config,
tags=self.tags,
logger_defs=self.logger_defs,
executor_def=self.executor_def,
hooks=self.hooks,
op_retry_policy=self.op_retry_policy,
version_strategy=self.version_strategy,
partitions_def=self.partitions_def,
)
update_wrapper(job_def, fn)
return job_def
def __call__(self, fn: Callable[..., Any]) -> PipelineDefinition:
check.callable_param(fn, "fn")
if not self.name:
self.name = fn.__name__
from dagster.core.definitions.decorators.composite_solid import do_composition
(
input_mappings,
output_mappings,
dependencies,
solid_defs,
config_mapping,
positional_inputs,
) = do_composition(
"@pipeline",
self.name,
fn,
self.input_defs,
self.output_defs,
self.config_schema,
self.config_fn,
ignore_output_from_composition_fn=not self.did_pass_outputs,
)
pipeline_def = PipelineDefinition(
name=self.name,
dependencies=dependencies,
solid_defs=solid_defs,
mode_defs=self.mode_definitions,
preset_defs=self.preset_definitions,
description=self.description,
tags=self.tags,
hook_defs=self.hook_defs,
input_mappings=input_mappings,
output_mappings=output_mappings,
config_mapping=config_mapping,
positional_inputs=positional_inputs,
solid_retry_policy=self.solid_retry_policy,
)
update_wrapper(pipeline_def, fn)
return pipeline_def
def __call__(self, fn: Callable[..., Any]) -> GraphDefinition:
check.callable_param(fn, "fn")
if not self.name:
self.name = fn.__name__
from dagster.core.definitions.decorators.composite_solid import do_composition
(
input_mappings,
output_mappings,
dependencies,
solid_defs,
config_mapping,
positional_inputs,
) = do_composition(
decorator_name="@graph",
graph_name=self.name,
fn=fn,
provided_input_defs=self.input_defs,
provided_output_defs=self.output_defs,
ignore_output_from_composition_fn=False,
config_schema=None,
config_fn=None,
)
graph_def = GraphDefinition(
name=self.name,
dependencies=dependencies,
node_defs=solid_defs,
description=self.description,
input_mappings=input_mappings,
output_mappings=output_mappings,
config_mapping=config_mapping,
positional_inputs=positional_inputs,
)
update_wrapper(graph_def, fn)
return graph_def