def __call__(self, fn: Callable[..., Any]) -> PipelineDefinition: check.callable_param(fn, "fn") if not self.name: self.name = fn.__name__ from dagster.core.definitions.decorators.composite_solid import ( do_composition, get_validated_config_mapping, ) config_mapping = get_validated_config_mapping( self.name, self.config_schema, self.config_fn, decorator_name="pipeline") ( input_mappings, output_mappings, dependencies, solid_defs, config_mapping, positional_inputs, ) = do_composition( "@pipeline", self.name, fn, self.input_defs, self.output_defs, config_mapping, ignore_output_from_composition_fn=not self.did_pass_outputs, ) pipeline_def = PipelineDefinition( mode_defs=self.mode_definitions, preset_defs=self.preset_definitions, graph_def=GraphDefinition( name=self.name, description=None, # put desc on the pipeline dependencies=dependencies, node_defs=solid_defs, input_mappings=input_mappings, output_mappings=output_mappings, config=config_mapping, positional_inputs=positional_inputs, ), tags=self.tags, description=self.description or format_docstring_for_description(fn), hook_defs=self.hook_defs, solid_retry_policy=self.solid_retry_policy, version_strategy=self.version_strategy, ) update_wrapper(pipeline_def, fn) return pipeline_def
def __call__(self, fn: Callable[..., Any]) -> GraphDefinition: check.callable_param(fn, "fn") if not self.name: self.name = fn.__name__ if self.ins is not None: input_defs = [inp.to_definition(name) for name, inp in self.ins.items()] else: input_defs = check.opt_list_param( self.input_defs, "input_defs", of_type=InputDefinition ) if self.out is None: output_defs = self.output_defs elif isinstance(self.out, GraphOut): output_defs = [self.out.to_definition(name=None)] else: check.dict_param(self.out, "out", key_type=str, value_type=GraphOut) output_defs = [out.to_definition(name=name) for name, out in self.out.items()] from dagster.core.definitions.decorators.composite_solid import do_composition ( input_mappings, output_mappings, dependencies, solid_defs, config_mapping, positional_inputs, ) = do_composition( decorator_name="@graph", graph_name=self.name, fn=fn, provided_input_defs=input_defs, provided_output_defs=output_defs, ignore_output_from_composition_fn=False, config_mapping=self.config_mapping, ) graph_def = GraphDefinition( name=self.name, dependencies=dependencies, node_defs=solid_defs, description=self.description or format_docstring_for_description(fn), input_mappings=input_mappings, output_mappings=output_mappings, config=config_mapping, positional_inputs=positional_inputs, tags=self.tags, ) update_wrapper(graph_def, fn) return graph_def
def __call__(self, fn: Callable[..., Any]) -> JobDefinition: check.callable_param(fn, "fn") if not self.name: self.name = fn.__name__ from dagster.core.definitions.decorators.composite_solid import do_composition ( input_mappings, output_mappings, dependencies, solid_defs, config_mapping, positional_inputs, ) = do_composition( decorator_name="@job", graph_name=self.name, fn=fn, provided_input_defs=[], provided_output_defs=[], ignore_output_from_composition_fn=False, config_mapping=None, ) graph_def = GraphDefinition( name=self.name, dependencies=dependencies, node_defs=solid_defs, description=self.description or format_docstring_for_description(fn), input_mappings=input_mappings, output_mappings=output_mappings, config=config_mapping, positional_inputs=positional_inputs, tags=self.tags, ) job_def = graph_def.to_job( description=self.description or format_docstring_for_description(fn), resource_defs=self.resource_defs, config=self.config, tags=self.tags, logger_defs=self.logger_defs, executor_def=self.executor_def, hooks=self.hooks, op_retry_policy=self.op_retry_policy, version_strategy=self.version_strategy, partitions_def=self.partitions_def, ) update_wrapper(job_def, fn) return job_def
def __call__(self, fn: Callable[..., Any]) -> PipelineDefinition: check.callable_param(fn, "fn") if not self.name: self.name = fn.__name__ from dagster.core.definitions.decorators.composite_solid import do_composition ( input_mappings, output_mappings, dependencies, solid_defs, config_mapping, positional_inputs, ) = do_composition( "@pipeline", self.name, fn, self.input_defs, self.output_defs, self.config_schema, self.config_fn, ignore_output_from_composition_fn=not self.did_pass_outputs, ) pipeline_def = PipelineDefinition( name=self.name, dependencies=dependencies, solid_defs=solid_defs, mode_defs=self.mode_definitions, preset_defs=self.preset_definitions, description=self.description, tags=self.tags, hook_defs=self.hook_defs, input_mappings=input_mappings, output_mappings=output_mappings, config_mapping=config_mapping, positional_inputs=positional_inputs, solid_retry_policy=self.solid_retry_policy, ) update_wrapper(pipeline_def, fn) return pipeline_def
def __call__(self, fn: Callable[..., Any]) -> GraphDefinition: check.callable_param(fn, "fn") if not self.name: self.name = fn.__name__ from dagster.core.definitions.decorators.composite_solid import do_composition ( input_mappings, output_mappings, dependencies, solid_defs, config_mapping, positional_inputs, ) = do_composition( decorator_name="@graph", graph_name=self.name, fn=fn, provided_input_defs=self.input_defs, provided_output_defs=self.output_defs, ignore_output_from_composition_fn=False, config_schema=None, config_fn=None, ) graph_def = GraphDefinition( name=self.name, dependencies=dependencies, node_defs=solid_defs, description=self.description, input_mappings=input_mappings, output_mappings=output_mappings, config_mapping=config_mapping, positional_inputs=positional_inputs, ) update_wrapper(graph_def, fn) return graph_def