def __new__(cls, dependencies): deps = check.list_param(dependencies, 'dependencies', of_type=DependencyDefinition) seen = {} for dep in deps: key = dep.solid + ':' + dep.output if key in seen: raise DagsterInvalidDefinitionError( 'Duplicate dependencies on solid "{dep.solid}" output "{dep.output}" ' 'used in the same MultiDependencyDefinition.'.format( dep=dep)) seen[key] = True return super(MultiDependencyDefinition, cls).__new__(cls, deps)
def _validate_in_mappings(input_mappings, solid_dict, name, class_name): input_def_dict = OrderedDict() for mapping in input_mappings: if isinstance(mapping, InputMapping): if input_def_dict.get(mapping.definition.name): if input_def_dict[ mapping.definition.name] != mapping.definition: raise DagsterInvalidDefinitionError( "In {class_name} {name} multiple input mappings with same " "definition name but different definitions".format( name=name, class_name=class_name), ) else: input_def_dict[mapping.definition.name] = mapping.definition target_solid = solid_dict.get(mapping.solid_name) if target_solid is None: raise DagsterInvalidDefinitionError( "In {class_name} '{name}' input mapping references solid " "'{solid_name}' which it does not contain.".format( name=name, solid_name=mapping.solid_name, class_name=class_name)) if not target_solid.has_input(mapping.input_name): raise DagsterInvalidDefinitionError( "In {class_name} '{name}' input mapping to solid '{mapping.solid_name}' " "which contains no input named '{mapping.input_name}'". format(name=name, mapping=mapping, class_name=class_name)) target_input = target_solid.input_def_named(mapping.input_name) if target_input.dagster_type != mapping.definition.dagster_type: raise DagsterInvalidDefinitionError( "In {class_name} '{name}' input " "'{mapping.definition.name}' of type {mapping.definition.dagster_type.display_name} maps to " "{mapping.solid_name}.{mapping.input_name} of different type " "{target_input.dagster_type.display_name}. InputMapping source and " "destination must have the same type.".format( mapping=mapping, name=name, target_input=target_input, class_name=class_name)) elif isinstance(mapping, InputDefinition): raise DagsterInvalidDefinitionError( "In {class_name} '{name}' you passed an InputDefinition " "named '{input_name}' directly in to input_mappings. Return " "an InputMapping by calling mapping_to on the InputDefinition." .format(name=name, input_name=mapping.name, class_name=class_name)) else: raise DagsterInvalidDefinitionError( "In {class_name} '{name}' received unexpected type '{type}' in input_mappings. " "Provide an OutputMapping using InputDefinition(...).mapping_to(...)" .format(type=type(mapping), name=name, class_name=class_name)) return input_mappings, input_def_dict.values()
def __init__(self, inner_type): inner_type = resolve_dagster_type(inner_type) if inner_type is Nothing: raise DagsterInvalidDefinitionError( 'Type Nothing can not be wrapped in List or Optional') key = 'Optional.' + inner_type.key self.inner_type = inner_type super(OptionalType, self).__init__( key=key, name=None, type_check_fn=self.type_check_method, input_hydration_config=_create_nullable_input_schema(inner_type), )
def _build_all_node_defs(node_defs): all_defs = {} for current_level_node_def in node_defs: for node_def in current_level_node_def.iterate_node_defs(): if node_def.name in all_defs: if all_defs[node_def.name] != node_def: raise DagsterInvalidDefinitionError( 'Detected conflicting solid definitions with the same name "{name}"'.format( name=node_def.name ) ) else: all_defs[node_def.name] = node_def return all_defs
def _check_default_value(input_name, dagster_type, default_value): if default_value is not _NoValueSentinel: if dagster_type.is_nothing: raise DagsterInvalidDefinitionError( "Setting a default_value is invalid on InputDefinitions of type Nothing" ) if isinstance(dagster_type, BuiltinScalarDagsterType): type_check = dagster_type.type_check_scalar_value(default_value) if not type_check.success: raise DagsterInvalidDefinitionError( ( "Type check failed for the default_value of InputDefinition " "{input_name} of type {dagster_type}. " "Received value {value} of type {type}" ).format( input_name=input_name, dagster_type=dagster_type.display_name, value=default_value, type=type(default_value), ), ) return default_value
def from_dict(repository_definitions): """Static constructor. Args: repository_definition (Dict[str, Dict[str, ...]]): A dict of the form: { 'pipelines': Dict[str, Callable[[], PipelineDefinition]], 'partition_sets': Dict[str, Callable[[], PartitionSetDefinition]], 'schedules': Dict[str, Callable[[], ScheduleDefinition]] } This form is intended to allow definitions to be created lazily when accessed by name, which can be helpful for performance when there are many definitions in a repository, or when constructing the definitions is costly. """ check.dict_param(repository_definitions, "repository_definitions", key_type=str) check.invariant( set(repository_definitions.keys()).issubset( VALID_REPOSITORY_DATA_DICT_KEYS), "Bad dict: must not contain keys other than {{{valid_keys}}}: found {bad_keys}." .format( valid_keys=", ".join([ "'{key}'".format(key=key) for key in VALID_REPOSITORY_DATA_DICT_KEYS ]), bad_keys=", ".join([ "'{key}'" for key in repository_definitions.keys() if key not in VALID_REPOSITORY_DATA_DICT_KEYS ]), ), ) for key in VALID_REPOSITORY_DATA_DICT_KEYS: if key not in repository_definitions: repository_definitions[key] = {} duplicate_keys = set( repository_definitions.get("schedules", {}).keys()).intersection( set(repository_definitions.get("sensors", {}).keys())) if duplicate_keys: raise DagsterInvalidDefinitionError( f"Duplicate definitions found for keys: {duplicate_keys.join(', ')}" ) return RepositoryData(**repository_definitions)
def __new__(cls, name, invocations, output_mapping_dict): dep_dict = {} solid_def_dict = {} input_mappings = [] for invocation in invocations.values(): def_name = invocation.solid_def.name if def_name in solid_def_dict and solid_def_dict[ def_name] is not invocation.solid_def: raise DagsterInvalidDefinitionError( 'Detected conflicting solid definitions with the same name "{name}"' .format(name=def_name)) solid_def_dict[def_name] = invocation.solid_def deps = {} for input_name, node in invocation.input_bindings.items(): if isinstance(node, InvokedSolidOutputHandle): deps[input_name] = DependencyDefinition( node.solid_name, node.output_name) elif isinstance(node, list) and all( map( lambda item: isinstance( item, InvokedSolidOutputHandle), node)): deps[input_name] = MultiDependencyDefinition([ DependencyDefinition(call.solid_name, call.output_name) for call in node ]) else: check.failed( "Unexpected input binding - got {node}".format( node=node)) dep_dict[SolidInvocation( invocation.solid_def.name, invocation.solid_name, tags=invocation.tags, hook_defs=invocation.hook_defs, )] = deps for input_name, node in invocation.input_mappings.items(): input_mappings.append( node.input_def.mapping_to(invocation.solid_name, input_name)) return super(cls, CompleteCompositionContext).__new__( cls, name, list(solid_def_dict.values()), dep_dict, input_mappings, output_mapping_dict)
def configured(self, config_or_config_fn, config_schema=None, **kwargs): """ Returns a new :py:class:`SolidDefinition` that bundles this definition with the specified config or config function. Args: config_or_config_fn (Union[Any, Callable[[Any], Any]]): Either (1) Run configuration that fully satisfies this solid's config schema or (2) A function that accepts run configuration and returns run configuration that fully satisfies this solid's config schema. In the latter case, config_schema must be specified. When passing a function, it's easiest to use :py:func:`configured`. config_schema (ConfigSchema): If config_or_config_fn is a function, the config schema that its input must satisfy. name (str): Name of the new (configured) solid. Must be unique within any :py:class:`PipelineDefinition` using the solid. Returns (SolidDefinition): A configured version of this solid definition. """ fn_name = config_or_config_fn.__name__ if callable(config_or_config_fn) else None name = kwargs.get("name", fn_name) if not name: raise DagsterInvalidDefinitionError( 'Missing string param "name" while attempting to configure the solid ' '"{solid_name}". When configuring a solid, you must specify a name for the ' "resulting solid definition as a keyword param or use `configured` in decorator " "form. For examples, visit https://docs.dagster.io/overview/configuration#configured.".format( solid_name=self.name ) ) wrapped_config_mapping_fn = self._get_wrapped_config_mapping_fn( config_or_config_fn, config_schema ) return SolidDefinition( name=name, input_defs=self.input_defs, compute_fn=self.compute_fn, output_defs=self.output_defs, config_schema=config_schema, description=kwargs.get("description", self.description), tags=self.tags, required_resource_keys=self.required_resource_keys, positional_inputs=self.positional_inputs, version=self.version, _configured_config_mapping_fn=wrapped_config_mapping_fn, )
def raise_bad_user_facing_field_argument(obj, param_name, error_context_str): from .field import resolve_to_config_type from .type_printer import print_config_type_to_string raise DagsterInvalidDefinitionError( ( 'You have passed an object {value_repr} of incorrect type ' '"{type_name}" in the parameter "{param_name}" ' '{error_context_str} where a Field, dict, or type was expected.' ).format( error_context_str=error_context_str, param_name=param_name, value_repr=repr(obj), type_name=type(obj).__name__, ) )
def resolve_config_field(config_field, config, source): if config_field is not None and config is not None: # https://github.com/dagster-io/dagster/issues/1974 # TODO: eliminate this once config_field is gone -- schrockn (12/10/2019) raise DagsterInvalidDefinitionError( 'Must only provide one of config_field or config but not both in {}.' 'Using the config arg is equivalent to config_field=Field(Dict(...)).' .format(source)) if config_field: return config_field if config: return resolve_config(config, source) return None
def _construct_solid_defs(self, pipelines): solid_defs = {} solid_to_pipeline = {} for pipeline in pipelines: for solid_def in pipeline.solid_defs: if solid_def.name not in solid_defs: solid_defs[solid_def.name] = solid_def solid_to_pipeline[solid_def.name] = pipeline.name elif self.enforce_uniqueness: if not solid_defs[solid_def.name] is solid_def: raise DagsterInvalidDefinitionError( 'Trying to add duplicate solid def {} in {}, Already saw in {}'.format( solid_def.name, pipeline.name, solid_to_pipeline[solid_def.name] ) ) return solid_defs
def infer_output_definitions(decorator_name, solid_name, fn): signature = funcsigs.signature(fn) try: description = _infer_output_description_from_docstring(fn) return [ OutputDefinition() if signature.return_annotation is funcsigs.Signature.empty else OutputDefinition(signature.return_annotation, description=description) ] except CheckError as type_error: raise DagsterInvalidDefinitionError( "Error inferring Dagster type for return type " f'"{signature.return_annotation}" from {decorator_name} "{solid_name}". ' f"Correct the issue or explicitly pass definitions to {decorator_name}." ) from type_error
def __init__(self, inner_type): inner_type = resolve_dagster_type(inner_type) if inner_type is Nothing: raise DagsterInvalidDefinitionError( "Type Nothing can not be wrapped in List or Optional") key = "Optional." + inner_type.key self.inner_type = inner_type super(OptionalType, self).__init__( key=key, name=None, kind=DagsterTypeKind.NULLABLE, type_check_fn=self.type_check_method, loader=_create_nullable_input_schema(inner_type), )
def wrapper(func): fn_positionals, _ = split_function_parameters(func, EXPECTED_POSITIONALS) missing_positional = validate_decorated_fn_positionals( fn_positionals, EXPECTED_POSITIONALS) if missing_positional: raise DagsterInvalidDefinitionError( "@dagster_type_loader '{solid_name}' decorated function does not have required positional " "parameter '{missing_param}'. Solid functions should only have keyword arguments " "that match input names and a first positional parameter named 'context'." .format(solid_name=func.__name__, missing_param=missing_positional)) return _create_type_loader_for_decorator(config_type, func, required_resource_keys, loader_version, external_version_fn)
def validate_dependency_dict(dependencies): prelude = ( 'The expected type for "dependencies" is dict[Union[str, SolidInvocation], dict[str, ' "DependencyDefinition]]. " ) if dependencies is None: return {} if not isinstance(dependencies, dict): raise DagsterInvalidDefinitionError( prelude + "Received value {val} of type {type} at the top level.".format( val=dependencies, type=type(dependencies) ) ) for key, dep_dict in dependencies.items(): if not (isinstance(key, six.string_types) or isinstance(key, SolidInvocation)): raise DagsterInvalidDefinitionError( prelude + "Expected str or SolidInvocation key in the top level dict. " "Received value {val} of type {type}".format(val=key, type=type(key)) ) if not isinstance(dep_dict, dict): if isinstance(dep_dict, IDependencyDefinition): raise DagsterInvalidDefinitionError( prelude + "Received a IDependencyDefinition one layer too high under key {key}. " "The DependencyDefinition should be moved in to a dict keyed on " "input name.".format(key=key) ) else: raise DagsterInvalidDefinitionError( prelude + "Under key {key} received value {val} of type {type}. " "Expected dict[str, DependencyDefinition]".format( key=key, val=dep_dict, type=type(dep_dict) ) ) for input_key, dep in dep_dict.items(): if not isinstance(input_key, six.string_types): raise DagsterInvalidDefinitionError( prelude + "Received non-sting key in the inner dict for key {key}.".format(key=key) ) if not isinstance(dep, IDependencyDefinition): raise DagsterInvalidDefinitionError( prelude + 'Expected IDependencyDefinition for solid "{key}" input "{input_key}". ' "Received value {val} of type {type}.".format( key=key, input_key=input_key, val=dep, type=type(dep) ) ) return dependencies
def _validate_dependencies(dependencies, solid_dict, alias_to_name): for from_solid, dep_by_input in dependencies.items(): for from_input, dep_def in dep_by_input.items(): for dep in dep_def.get_definitions(): if from_solid == dep.solid: raise DagsterInvalidDefinitionError(( 'Invalid dependencies: circular reference detected in solid "{from_solid}" input "{from_input}"' ).format(from_solid=from_solid, from_input=from_input)) if not from_solid in solid_dict: aliased_solid = alias_to_name.get(from_solid) if aliased_solid == from_solid: raise DagsterInvalidDefinitionError( 'Invalid dependencies: solid "{solid}" in dependency dictionary not found in solid list' .format(solid=from_solid)) else: raise DagsterInvalidDefinitionError(( 'Invalid dependencies: solid "{aliased_solid}" (aliased by "{from_solid}" in dependency ' 'dictionary) not found in solid list').format( aliased_solid=aliased_solid, from_solid=from_solid)) if not solid_dict[from_solid].definition.has_input(from_input): input_list = solid_dict[ from_solid].definition.input_dict.keys() raise DagsterInvalidDefinitionError( 'Invalid dependencies: solid "{from_solid}" does not have input "{from_input}". ' .format(from_solid=from_solid, from_input=from_input) + 'Available inputs: {input_list}'.format( input_list=input_list)) if not dep.solid in solid_dict: raise DagsterInvalidDefinitionError( 'Invalid dependencies: solid "{dep.solid}" not found in solid list. ' 'Listed as dependency for solid "{from_solid}" input "{from_input}" ' .format(dep=dep, from_solid=from_solid, from_input=from_input)) if not solid_dict[dep.solid].definition.has_output(dep.output): raise DagsterInvalidDefinitionError( 'Invalid dependencies: solid "{dep.solid}" does not have output "{dep.output}". ' 'Listed as dependency for solid "{from_solid} input "{from_input}"' .format(dep=dep, from_solid=from_solid, from_input=from_input)) input_def = solid_dict[from_solid].definition.input_def_named( from_input) output_def = solid_dict[dep.solid].definition.output_def_named( dep.output) _validate_input_output_pair(input_def, output_def, from_solid, dep)
def construct_dagster_type_dictionary(solid_defs): type_dict = {t.name: t for t in ALL_RUNTIME_BUILTINS} for solid_def in solid_defs: for runtime_type in solid_def.all_runtime_types(): if not runtime_type.name: continue if runtime_type.name not in type_dict: type_dict[runtime_type.name] = runtime_type continue if type_dict[runtime_type.name] is not runtime_type: raise DagsterInvalidDefinitionError(( 'You have created two dagster types with the same name "{type_name}". ' 'Dagster types have must have unique names.').format( type_name=runtime_type.name)) return type_dict
def from_files(name, environment_files=None, solid_subset=None, mode=None): '''Static constructor for presets from YAML files. Args: name (str): The name of this preset. Must be unique in the presets defined on a given pipeline. environment_files (Optional[List[str]]): List of paths or glob patterns for yaml files to load and parse as the environment config for this preset. solid_subset (Optional[List[str]]): The list of names of solid invocations (i.e., of unaliased solids or of their aliases if aliased) to execute with this preset. mode (Optional[str]): The mode to apply when executing this preset. (default: 'default') ''' check.str_param(name, 'name') environment_files = check.opt_list_param(environment_files, 'environment_files') solid_subset = check.opt_nullable_list_param(solid_subset, 'solid_subset', of_type=str) mode = check.opt_str_param(mode, 'mode', DEFAULT_MODE_NAME) file_set = set() for file_glob in environment_files or []: files = glob(file_glob) if not files: raise DagsterInvalidDefinitionError( 'File or glob pattern "{file_glob}" for "environment_files" in preset ' '"{name}" produced no results.'.format( name=name, file_glob=file_glob)) file_set.update(map(os.path.realpath, files)) try: merged = merge_yamls(list(file_set)) except yaml.YAMLError as err: six.raise_from( DagsterInvariantViolationError( 'Encountered error attempting to parse yaml. Parsing files {file_set} ' 'loaded by file/patterns {files} on preset "{name}".'. format(file_set=file_set, files=environment_files, name=name)), err, ) return PresetDefinition(name, merged, solid_subset, mode)
def _name_for_configured_node( self, old_name: Optional[str], new_name: Optional[str], original_config_or_config_fn: Optional[Callable], ) -> Optional[str]: fn_name = (original_config_or_config_fn.__name__ if callable(original_config_or_config_fn) else None) name = new_name or fn_name if not name: raise DagsterInvalidDefinitionError( 'Missing string param "name" while attempting to configure the node ' '"{node_name}". When configuring a node, you must specify a name for the ' "resulting node definition as a keyword param or use `configured` in decorator " "form. For examples, visit https://docs.dagster.io/overview/configuration#configured." .format(node_name=old_name, )) return name
def _validate_inputs(dependency_structure, solid_dict): for solid in solid_dict.values(): for handle in solid.input_handles(): if not dependency_structure.has_deps(handle): if (not handle.input_def.runtime_type.input_hydration_config and not handle.input_def.runtime_type.is_nothing): raise DagsterInvalidDefinitionError( 'Input "{input_name}" in solid "{solid_name}" is not connected to ' 'any outputs and can not be hydrated from configuration, creating an impossible to execute pipeline. ' 'Posible solutions are:\n' ' * add a input_hydration_config for the type "{runtime_type}"\n' ' * connect "{input_name}" to the output of another solid\n' .format( solid_name=solid.name, input_name=handle.input_def.name, runtime_type=handle.input_def.runtime_type.name, ))
def __init__( self, name, cron_schedule, pipeline_name, environment_dict=None, environment_dict_fn=None, tags=None, mode="default", should_execute=lambda: True, environment_vars=None, ): self._schedule_definition_data = ScheduleDefinitionData( name=check.str_param(name, 'name'), cron_schedule=check.str_param(cron_schedule, 'cron_schedule'), environment_vars=check.opt_dict_param(environment_vars, 'environment_vars'), ) check.str_param(pipeline_name, 'pipeline_name') environment_dict = check.opt_dict_param(environment_dict, 'environment_dict') tags = check.opt_list_param(tags, 'tags') check.str_param(mode, 'mode') self._environment_dict_fn = check.opt_callable_param( environment_dict_fn, 'environment_dict_fn') self._should_execute = check.callable_param(should_execute, 'should_execute') if self._environment_dict_fn and environment_dict: raise DagsterInvalidDefinitionError( 'Attempted to provide both environment_dict_fn and environment_dict as arguments' ' to ScheduleDefinition. Must provide only one of the two.') self._execution_params = { 'environmentConfigData': environment_dict, 'selector': { 'name': pipeline_name }, 'executionMetadata': { "tags": tags }, 'mode': mode, }
def _validate_input_output_pair(input_def, output_def, from_solid, dep): # Currently, we opt to be overly permissive with input/output type mismatches. # Here we check for the case where no value will be provided where one is expected. if output_def.runtime_type.is_nothing and not input_def.runtime_type.is_nothing: raise DagsterInvalidDefinitionError(( 'Input "{input_def.name}" to solid "{from_solid}" can not depend on the output ' '"{output_def.name}" from solid "{dep.solid}". ' 'Input "{input_def.name}" expects a value of type ' '{input_def.runtime_type.display_name} and output "{output_def.name}" returns ' 'type {output_def.runtime_type.display_name}{extra}.').format( from_solid=from_solid, dep=dep, output_def=output_def, input_def=input_def, extra=' (which produces no value)' if output_def.runtime_type.is_nothing else '', ))
def _validate_resource_dependencies(mode_definitions, solids): '''This validation ensures that each pipeline context provides the resources that are required by each solid. ''' check.list_param(mode_definitions, 'mode_definintions', of_type=ModeDefinition) check.list_param(solids, 'solids', of_type=ISolidDefinition) for mode_def in mode_definitions: mode_resources = set(mode_def.resource_defs.keys()) for solid in solids: for resource in solid.resources: if resource not in mode_resources: raise DagsterInvalidDefinitionError( ( 'Resource "{resource}" is required by solid {solid_name}, but is not ' 'provided by mode "{mode_name}"' ).format(resource=resource, solid_name=solid.name, mode_name=mode_def.name) )
def __init__(self, inner_type: DagsterType): inner_type = resolve_dagster_type(inner_type) if inner_type is Nothing: raise DagsterInvalidDefinitionError( "Type Nothing can not be wrapped in List or Optional") key = "Optional." + cast(str, inner_type.key) self.inner_type = inner_type super(OptionalType, self).__init__( key=key, name=None, kind=DagsterTypeKind.NULLABLE, type_check_fn=self.type_check_method, loader=_create_nullable_input_schema(inner_type), # This throws a type error with Py typing_type=t.Optional[inner_type.typing_type], # type: ignore )
def make_python_type_usable_as_dagster_type(python_type, dagster_type): ''' Take any existing python type and map it to a dagster type (generally created with :py:class:`DagsterType <dagster.DagsterType>`) This can only be called once on a given python type. ''' check.inst_param(dagster_type, 'dagster_type', DagsterType) if python_type in _PYTHON_TYPE_TO_DAGSTER_TYPE_MAPPING_REGISTRY: # This would be just a great place to insert a short URL pointing to the type system # documentation into the error message # https://github.com/dagster-io/dagster/issues/1831 raise DagsterInvalidDefinitionError(( 'A Dagster type has already been registered for the Python type ' '{python_type}. make_python_type_usable_as_dagster_type can only ' 'be called once on a python type as it is registering a 1:1 mapping ' 'between that python type and a dagster type.').format( python_type=python_type)) _PYTHON_TYPE_TO_DAGSTER_TYPE_MAPPING_REGISTRY[python_type] = dagster_type
def _validate_inputs(dependency_structure, solid_dict): for solid in solid_dict.values(): for handle in solid.input_handles(): if not dependency_structure.has_deps(handle): if (not handle.input_def.dagster_type.input_hydration_config and not handle.input_def.dagster_type.kind == DagsterTypeKind.NOTHING): raise DagsterInvalidDefinitionError( 'Input "{input_name}" in solid "{solid_name}" is not connected to ' 'the output of a previous solid and can not be hydrated from configuration, ' 'creating an impossible to execute pipeline. ' 'Possible solutions are:\n' ' * add a input_hydration_config for the type "{dagster_type}"\n' ' * connect "{input_name}" to the output of another solid\n' .format( solid_name=solid.name, input_name=handle.input_def.name, dagster_type=handle.input_def.dagster_type. display_name, ))
def __new__(cls, dependencies): from .composition import MappedInputPlaceholder deps = check.list_param(dependencies, "dependencies") seen = {} for dep in deps: if isinstance(dep, DependencyDefinition): key = dep.solid + ":" + dep.output if key in seen: raise DagsterInvalidDefinitionError( 'Duplicate dependencies on solid "{dep.solid}" output "{dep.output}" ' "used in the same MultiDependencyDefinition.".format(dep=dep) ) seen[key] = True elif dep is MappedInputPlaceholder: pass else: check.failed("Unexpected dependencies entry {}".format(dep)) return super(MultiDependencyDefinition, cls).__new__(cls, deps)
def _validate_input_output_pair(input_def, output_def, from_solid, dep): # Currently, we opt to be overly permissive with input/output type mismatches. # Here we check for the case where no value will be provided where one is expected. if (output_def.dagster_type.kind == DagsterTypeKind.NOTHING and not input_def.dagster_type.kind == DagsterTypeKind.NOTHING): raise DagsterInvalidDefinitionError(( 'Input "{input_def.name}" to node "{from_solid}" can not depend on the output ' '"{output_def.name}" from node "{dep.solid}". ' 'Input "{input_def.name}" expects a value of type ' '{input_def.dagster_type.display_name} and output "{output_def.name}" returns ' "type {output_def.dagster_type.display_name}{extra}.").format( from_solid=from_solid, dep=dep, output_def=output_def, input_def=input_def, extra=" (which produces no value)" if output_def.dagster_type.kind == DagsterTypeKind.NOTHING else "", ))
def _check_solid_defs(self): solid_defs = {} solid_to_pipeline = {} for pipeline in self._all_pipelines: for solid_def in pipeline.all_node_defs + [pipeline.graph]: if solid_def.name not in solid_defs: solid_defs[solid_def.name] = solid_def solid_to_pipeline[solid_def.name] = pipeline.name if not solid_defs[solid_def.name] is solid_def: first_name, second_name = sorted( [solid_to_pipeline[solid_def.name], pipeline.name] ) raise DagsterInvalidDefinitionError( ( f"Conflicting definitions found in repository with name '{solid_def.name}'. " "Op/Graph/Solid definition names must be unique within a " f"repository. {solid_def.__class__.__name__} is defined in {pipeline.target_type} " f"'{first_name}' and in {pipeline.target_type} '{second_name}'." ) )
def _infer_inputs_from_params(params, decorator_name, solid_name): input_defs = [] for param in params: try: input_defs.append(InputDefinition(param.name, _input_param_type(param.annotation))) except DagsterInvalidDefinitionError as type_error: six.raise_from( DagsterInvalidDefinitionError( 'Error inferring Dagster type for input name {param} typed as ' '"{type_annotation}" from {decorator} "{solid}". ' 'Correct the issue or explicitly pass definitions to {decorator}.'.format( decorator=decorator_name, solid=solid_name, param=param.name, type_annotation=param.annotation, ) ), type_error, ) return input_defs