def __new__(cls, dependencies):
deps = check.list_param(dependencies,
'dependencies',
of_type=DependencyDefinition)
seen = {}
for dep in deps:
key = dep.solid + ':' + dep.output
if key in seen:
raise DagsterInvalidDefinitionError(
'Duplicate dependencies on solid "{dep.solid}" output "{dep.output}" '
'used in the same MultiDependencyDefinition.'.format(
dep=dep))
seen[key] = True
return super(MultiDependencyDefinition, cls).__new__(cls, deps)
def _validate_in_mappings(input_mappings, solid_dict, name, class_name):
input_def_dict = OrderedDict()
for mapping in input_mappings:
if isinstance(mapping, InputMapping):
if input_def_dict.get(mapping.definition.name):
if input_def_dict[
mapping.definition.name] != mapping.definition:
raise DagsterInvalidDefinitionError(
"In {class_name} {name} multiple input mappings with same "
"definition name but different definitions".format(
name=name, class_name=class_name), )
else:
input_def_dict[mapping.definition.name] = mapping.definition
target_solid = solid_dict.get(mapping.solid_name)
if target_solid is None:
raise DagsterInvalidDefinitionError(
"In {class_name} '{name}' input mapping references solid "
"'{solid_name}' which it does not contain.".format(
name=name,
solid_name=mapping.solid_name,
class_name=class_name))
if not target_solid.has_input(mapping.input_name):
raise DagsterInvalidDefinitionError(
"In {class_name} '{name}' input mapping to solid '{mapping.solid_name}' "
"which contains no input named '{mapping.input_name}'".
format(name=name, mapping=mapping, class_name=class_name))
target_input = target_solid.input_def_named(mapping.input_name)
if target_input.dagster_type != mapping.definition.dagster_type:
raise DagsterInvalidDefinitionError(
"In {class_name} '{name}' input "
"'{mapping.definition.name}' of type {mapping.definition.dagster_type.display_name} maps to "
"{mapping.solid_name}.{mapping.input_name} of different type "
"{target_input.dagster_type.display_name}. InputMapping source and "
"destination must have the same type.".format(
mapping=mapping,
name=name,
target_input=target_input,
class_name=class_name))
elif isinstance(mapping, InputDefinition):
raise DagsterInvalidDefinitionError(
"In {class_name} '{name}' you passed an InputDefinition "
"named '{input_name}' directly in to input_mappings. Return "
"an InputMapping by calling mapping_to on the InputDefinition."
.format(name=name,
input_name=mapping.name,
class_name=class_name))
else:
raise DagsterInvalidDefinitionError(
"In {class_name} '{name}' received unexpected type '{type}' in input_mappings. "
"Provide an OutputMapping using InputDefinition(...).mapping_to(...)"
.format(type=type(mapping), name=name, class_name=class_name))
return input_mappings, input_def_dict.values()
def __init__(self, inner_type):
inner_type = resolve_dagster_type(inner_type)
if inner_type is Nothing:
raise DagsterInvalidDefinitionError(
'Type Nothing can not be wrapped in List or Optional')
key = 'Optional.' + inner_type.key
self.inner_type = inner_type
super(OptionalType, self).__init__(
key=key,
name=None,
type_check_fn=self.type_check_method,
input_hydration_config=_create_nullable_input_schema(inner_type),
)
def _build_all_node_defs(node_defs):
all_defs = {}
for current_level_node_def in node_defs:
for node_def in current_level_node_def.iterate_node_defs():
if node_def.name in all_defs:
if all_defs[node_def.name] != node_def:
raise DagsterInvalidDefinitionError(
'Detected conflicting solid definitions with the same name "{name}"'.format(
name=node_def.name
)
)
else:
all_defs[node_def.name] = node_def
return all_defs
def _check_default_value(input_name, dagster_type, default_value):
if default_value is not _NoValueSentinel:
if dagster_type.is_nothing:
raise DagsterInvalidDefinitionError(
"Setting a default_value is invalid on InputDefinitions of type Nothing"
)
if isinstance(dagster_type, BuiltinScalarDagsterType):
type_check = dagster_type.type_check_scalar_value(default_value)
if not type_check.success:
raise DagsterInvalidDefinitionError(
(
"Type check failed for the default_value of InputDefinition "
"{input_name} of type {dagster_type}. "
"Received value {value} of type {type}"
).format(
input_name=input_name,
dagster_type=dagster_type.display_name,
value=default_value,
type=type(default_value),
),
)
return default_value
def from_dict(repository_definitions):
"""Static constructor.
Args:
repository_definition (Dict[str, Dict[str, ...]]): A dict of the form:
{
'pipelines': Dict[str, Callable[[], PipelineDefinition]],
'partition_sets': Dict[str, Callable[[], PartitionSetDefinition]],
'schedules': Dict[str, Callable[[], ScheduleDefinition]]
}
This form is intended to allow definitions to be created lazily when accessed by name,
which can be helpful for performance when there are many definitions in a repository, or
when constructing the definitions is costly.
"""
check.dict_param(repository_definitions,
"repository_definitions",
key_type=str)
check.invariant(
set(repository_definitions.keys()).issubset(
VALID_REPOSITORY_DATA_DICT_KEYS),
"Bad dict: must not contain keys other than {{{valid_keys}}}: found {bad_keys}."
.format(
valid_keys=", ".join([
"'{key}'".format(key=key)
for key in VALID_REPOSITORY_DATA_DICT_KEYS
]),
bad_keys=", ".join([
"'{key}'" for key in repository_definitions.keys()
if key not in VALID_REPOSITORY_DATA_DICT_KEYS
]),
),
)
for key in VALID_REPOSITORY_DATA_DICT_KEYS:
if key not in repository_definitions:
repository_definitions[key] = {}
duplicate_keys = set(
repository_definitions.get("schedules", {}).keys()).intersection(
set(repository_definitions.get("sensors", {}).keys()))
if duplicate_keys:
raise DagsterInvalidDefinitionError(
f"Duplicate definitions found for keys: {duplicate_keys.join(', ')}"
)
return RepositoryData(**repository_definitions)
def __new__(cls, name, invocations, output_mapping_dict):
dep_dict = {}
solid_def_dict = {}
input_mappings = []
for invocation in invocations.values():
def_name = invocation.solid_def.name
if def_name in solid_def_dict and solid_def_dict[
def_name] is not invocation.solid_def:
raise DagsterInvalidDefinitionError(
'Detected conflicting solid definitions with the same name "{name}"'
.format(name=def_name))
solid_def_dict[def_name] = invocation.solid_def
deps = {}
for input_name, node in invocation.input_bindings.items():
if isinstance(node, InvokedSolidOutputHandle):
deps[input_name] = DependencyDefinition(
node.solid_name, node.output_name)
elif isinstance(node, list) and all(
map(
lambda item: isinstance(
item, InvokedSolidOutputHandle), node)):
deps[input_name] = MultiDependencyDefinition([
DependencyDefinition(call.solid_name, call.output_name)
for call in node
])
else:
check.failed(
"Unexpected input binding - got {node}".format(
node=node))
dep_dict[SolidInvocation(
invocation.solid_def.name,
invocation.solid_name,
tags=invocation.tags,
hook_defs=invocation.hook_defs,
)] = deps
for input_name, node in invocation.input_mappings.items():
input_mappings.append(
node.input_def.mapping_to(invocation.solid_name,
input_name))
return super(cls, CompleteCompositionContext).__new__(
cls, name, list(solid_def_dict.values()), dep_dict, input_mappings,
output_mapping_dict)
def configured(self, config_or_config_fn, config_schema=None, **kwargs):
"""
Returns a new :py:class:`SolidDefinition` that bundles this definition with the specified
config or config function.
Args:
config_or_config_fn (Union[Any, Callable[[Any], Any]]): Either (1) Run configuration
that fully satisfies this solid's config schema or (2) A function that accepts run
configuration and returns run configuration that fully satisfies this solid's
config schema. In the latter case, config_schema must be specified. When
passing a function, it's easiest to use :py:func:`configured`.
config_schema (ConfigSchema): If config_or_config_fn is a function, the config schema
that its input must satisfy.
name (str): Name of the new (configured) solid. Must be unique within any
:py:class:`PipelineDefinition` using the solid.
Returns (SolidDefinition): A configured version of this solid definition.
"""
fn_name = config_or_config_fn.__name__ if callable(config_or_config_fn) else None
name = kwargs.get("name", fn_name)
if not name:
raise DagsterInvalidDefinitionError(
'Missing string param "name" while attempting to configure the solid '
'"{solid_name}". When configuring a solid, you must specify a name for the '
"resulting solid definition as a keyword param or use `configured` in decorator "
"form. For examples, visit https://docs.dagster.io/overview/configuration#configured.".format(
solid_name=self.name
)
)
wrapped_config_mapping_fn = self._get_wrapped_config_mapping_fn(
config_or_config_fn, config_schema
)
return SolidDefinition(
name=name,
input_defs=self.input_defs,
compute_fn=self.compute_fn,
output_defs=self.output_defs,
config_schema=config_schema,
description=kwargs.get("description", self.description),
tags=self.tags,
required_resource_keys=self.required_resource_keys,
positional_inputs=self.positional_inputs,
version=self.version,
_configured_config_mapping_fn=wrapped_config_mapping_fn,
)
def raise_bad_user_facing_field_argument(obj, param_name, error_context_str):
from .field import resolve_to_config_type
from .type_printer import print_config_type_to_string
raise DagsterInvalidDefinitionError(
(
'You have passed an object {value_repr} of incorrect type '
'"{type_name}" in the parameter "{param_name}" '
'{error_context_str} where a Field, dict, or type was expected.'
).format(
error_context_str=error_context_str,
param_name=param_name,
value_repr=repr(obj),
type_name=type(obj).__name__,
)
)
def resolve_config_field(config_field, config, source):
if config_field is not None and config is not None:
# https://github.com/dagster-io/dagster/issues/1974
# TODO: eliminate this once config_field is gone -- schrockn (12/10/2019)
raise DagsterInvalidDefinitionError(
'Must only provide one of config_field or config but not both in {}.'
'Using the config arg is equivalent to config_field=Field(Dict(...)).'
.format(source))
if config_field:
return config_field
if config:
return resolve_config(config, source)
return None
def _construct_solid_defs(self, pipelines):
solid_defs = {}
solid_to_pipeline = {}
for pipeline in pipelines:
for solid_def in pipeline.solid_defs:
if solid_def.name not in solid_defs:
solid_defs[solid_def.name] = solid_def
solid_to_pipeline[solid_def.name] = pipeline.name
elif self.enforce_uniqueness:
if not solid_defs[solid_def.name] is solid_def:
raise DagsterInvalidDefinitionError(
'Trying to add duplicate solid def {} in {}, Already saw in {}'.format(
solid_def.name, pipeline.name, solid_to_pipeline[solid_def.name]
)
)
return solid_defs
def infer_output_definitions(decorator_name, solid_name, fn):
signature = funcsigs.signature(fn)
try:
description = _infer_output_description_from_docstring(fn)
return [
OutputDefinition()
if signature.return_annotation is funcsigs.Signature.empty
else OutputDefinition(signature.return_annotation, description=description)
]
except CheckError as type_error:
raise DagsterInvalidDefinitionError(
"Error inferring Dagster type for return type "
f'"{signature.return_annotation}" from {decorator_name} "{solid_name}". '
f"Correct the issue or explicitly pass definitions to {decorator_name}."
) from type_error
def __init__(self, inner_type):
inner_type = resolve_dagster_type(inner_type)
if inner_type is Nothing:
raise DagsterInvalidDefinitionError(
"Type Nothing can not be wrapped in List or Optional")
key = "Optional." + inner_type.key
self.inner_type = inner_type
super(OptionalType, self).__init__(
key=key,
name=None,
kind=DagsterTypeKind.NULLABLE,
type_check_fn=self.type_check_method,
loader=_create_nullable_input_schema(inner_type),
)
def wrapper(func):
fn_positionals, _ = split_function_parameters(func,
EXPECTED_POSITIONALS)
missing_positional = validate_decorated_fn_positionals(
fn_positionals, EXPECTED_POSITIONALS)
if missing_positional:
raise DagsterInvalidDefinitionError(
"@dagster_type_loader '{solid_name}' decorated function does not have required positional "
"parameter '{missing_param}'. Solid functions should only have keyword arguments "
"that match input names and a first positional parameter named 'context'."
.format(solid_name=func.__name__,
missing_param=missing_positional))
return _create_type_loader_for_decorator(config_type, func,
required_resource_keys,
loader_version,
external_version_fn)
def validate_dependency_dict(dependencies):
prelude = (
'The expected type for "dependencies" is dict[Union[str, SolidInvocation], dict[str, '
"DependencyDefinition]]. "
)
if dependencies is None:
return {}
if not isinstance(dependencies, dict):
raise DagsterInvalidDefinitionError(
prelude
+ "Received value {val} of type {type} at the top level.".format(
val=dependencies, type=type(dependencies)
)
)
for key, dep_dict in dependencies.items():
if not (isinstance(key, six.string_types) or isinstance(key, SolidInvocation)):
raise DagsterInvalidDefinitionError(
prelude + "Expected str or SolidInvocation key in the top level dict. "
"Received value {val} of type {type}".format(val=key, type=type(key))
)
if not isinstance(dep_dict, dict):
if isinstance(dep_dict, IDependencyDefinition):
raise DagsterInvalidDefinitionError(
prelude
+ "Received a IDependencyDefinition one layer too high under key {key}. "
"The DependencyDefinition should be moved in to a dict keyed on "
"input name.".format(key=key)
)
else:
raise DagsterInvalidDefinitionError(
prelude + "Under key {key} received value {val} of type {type}. "
"Expected dict[str, DependencyDefinition]".format(
key=key, val=dep_dict, type=type(dep_dict)
)
)
for input_key, dep in dep_dict.items():
if not isinstance(input_key, six.string_types):
raise DagsterInvalidDefinitionError(
prelude
+ "Received non-sting key in the inner dict for key {key}.".format(key=key)
)
if not isinstance(dep, IDependencyDefinition):
raise DagsterInvalidDefinitionError(
prelude
+ 'Expected IDependencyDefinition for solid "{key}" input "{input_key}". '
"Received value {val} of type {type}.".format(
key=key, input_key=input_key, val=dep, type=type(dep)
)
)
return dependencies
def _validate_dependencies(dependencies, solid_dict, alias_to_name):
for from_solid, dep_by_input in dependencies.items():
for from_input, dep_def in dep_by_input.items():
for dep in dep_def.get_definitions():
if from_solid == dep.solid:
raise DagsterInvalidDefinitionError((
'Invalid dependencies: circular reference detected in solid "{from_solid}" input "{from_input}"'
).format(from_solid=from_solid, from_input=from_input))
if not from_solid in solid_dict:
aliased_solid = alias_to_name.get(from_solid)
if aliased_solid == from_solid:
raise DagsterInvalidDefinitionError(
'Invalid dependencies: solid "{solid}" in dependency dictionary not found in solid list'
.format(solid=from_solid))
else:
raise DagsterInvalidDefinitionError((
'Invalid dependencies: solid "{aliased_solid}" (aliased by "{from_solid}" in dependency '
'dictionary) not found in solid list').format(
aliased_solid=aliased_solid,
from_solid=from_solid))
if not solid_dict[from_solid].definition.has_input(from_input):
input_list = solid_dict[
from_solid].definition.input_dict.keys()
raise DagsterInvalidDefinitionError(
'Invalid dependencies: solid "{from_solid}" does not have input "{from_input}". '
.format(from_solid=from_solid, from_input=from_input) +
'Available inputs: {input_list}'.format(
input_list=input_list))
if not dep.solid in solid_dict:
raise DagsterInvalidDefinitionError(
'Invalid dependencies: solid "{dep.solid}" not found in solid list. '
'Listed as dependency for solid "{from_solid}" input "{from_input}" '
.format(dep=dep,
from_solid=from_solid,
from_input=from_input))
if not solid_dict[dep.solid].definition.has_output(dep.output):
raise DagsterInvalidDefinitionError(
'Invalid dependencies: solid "{dep.solid}" does not have output "{dep.output}". '
'Listed as dependency for solid "{from_solid} input "{from_input}"'
.format(dep=dep,
from_solid=from_solid,
from_input=from_input))
input_def = solid_dict[from_solid].definition.input_def_named(
from_input)
output_def = solid_dict[dep.solid].definition.output_def_named(
dep.output)
_validate_input_output_pair(input_def, output_def, from_solid,
dep)
def construct_dagster_type_dictionary(solid_defs):
type_dict = {t.name: t for t in ALL_RUNTIME_BUILTINS}
for solid_def in solid_defs:
for runtime_type in solid_def.all_runtime_types():
if not runtime_type.name:
continue
if runtime_type.name not in type_dict:
type_dict[runtime_type.name] = runtime_type
continue
if type_dict[runtime_type.name] is not runtime_type:
raise DagsterInvalidDefinitionError((
'You have created two dagster types with the same name "{type_name}". '
'Dagster types have must have unique names.').format(
type_name=runtime_type.name))
return type_dict
def from_files(name, environment_files=None, solid_subset=None, mode=None):
'''Static constructor for presets from YAML files.
Args:
name (str): The name of this preset. Must be unique in the presets defined on a given
pipeline.
environment_files (Optional[List[str]]): List of paths or glob patterns for yaml files
to load and parse as the environment config for this preset.
solid_subset (Optional[List[str]]): The list of names of solid invocations (i.e., of
unaliased solids or of their aliases if aliased) to execute with this preset.
mode (Optional[str]): The mode to apply when executing this preset. (default:
'default')
'''
check.str_param(name, 'name')
environment_files = check.opt_list_param(environment_files,
'environment_files')
solid_subset = check.opt_nullable_list_param(solid_subset,
'solid_subset',
of_type=str)
mode = check.opt_str_param(mode, 'mode', DEFAULT_MODE_NAME)
file_set = set()
for file_glob in environment_files or []:
files = glob(file_glob)
if not files:
raise DagsterInvalidDefinitionError(
'File or glob pattern "{file_glob}" for "environment_files" in preset '
'"{name}" produced no results.'.format(
name=name, file_glob=file_glob))
file_set.update(map(os.path.realpath, files))
try:
merged = merge_yamls(list(file_set))
except yaml.YAMLError as err:
six.raise_from(
DagsterInvariantViolationError(
'Encountered error attempting to parse yaml. Parsing files {file_set} '
'loaded by file/patterns {files} on preset "{name}".'.
format(file_set=file_set,
files=environment_files,
name=name)),
err,
)
return PresetDefinition(name, merged, solid_subset, mode)
def _name_for_configured_node(
self,
old_name: Optional[str],
new_name: Optional[str],
original_config_or_config_fn: Optional[Callable],
) -> Optional[str]:
fn_name = (original_config_or_config_fn.__name__
if callable(original_config_or_config_fn) else None)
name = new_name or fn_name
if not name:
raise DagsterInvalidDefinitionError(
'Missing string param "name" while attempting to configure the node '
'"{node_name}". When configuring a node, you must specify a name for the '
"resulting node definition as a keyword param or use `configured` in decorator "
"form. For examples, visit https://docs.dagster.io/overview/configuration#configured."
.format(node_name=old_name, ))
return name
def _validate_inputs(dependency_structure, solid_dict):
for solid in solid_dict.values():
for handle in solid.input_handles():
if not dependency_structure.has_deps(handle):
if (not handle.input_def.runtime_type.input_hydration_config
and not handle.input_def.runtime_type.is_nothing):
raise DagsterInvalidDefinitionError(
'Input "{input_name}" in solid "{solid_name}" is not connected to '
'any outputs and can not be hydrated from configuration, creating an impossible to execute pipeline. '
'Posible solutions are:\n'
' * add a input_hydration_config for the type "{runtime_type}"\n'
' * connect "{input_name}" to the output of another solid\n'
.format(
solid_name=solid.name,
input_name=handle.input_def.name,
runtime_type=handle.input_def.runtime_type.name,
))
def __init__(
self,
name,
cron_schedule,
pipeline_name,
environment_dict=None,
environment_dict_fn=None,
tags=None,
mode="default",
should_execute=lambda: True,
environment_vars=None,
):
self._schedule_definition_data = ScheduleDefinitionData(
name=check.str_param(name, 'name'),
cron_schedule=check.str_param(cron_schedule, 'cron_schedule'),
environment_vars=check.opt_dict_param(environment_vars,
'environment_vars'),
)
check.str_param(pipeline_name, 'pipeline_name')
environment_dict = check.opt_dict_param(environment_dict,
'environment_dict')
tags = check.opt_list_param(tags, 'tags')
check.str_param(mode, 'mode')
self._environment_dict_fn = check.opt_callable_param(
environment_dict_fn, 'environment_dict_fn')
self._should_execute = check.callable_param(should_execute,
'should_execute')
if self._environment_dict_fn and environment_dict:
raise DagsterInvalidDefinitionError(
'Attempted to provide both environment_dict_fn and environment_dict as arguments'
' to ScheduleDefinition. Must provide only one of the two.')
self._execution_params = {
'environmentConfigData': environment_dict,
'selector': {
'name': pipeline_name
},
'executionMetadata': {
"tags": tags
},
'mode': mode,
}
def _validate_input_output_pair(input_def, output_def, from_solid, dep):
# Currently, we opt to be overly permissive with input/output type mismatches.
# Here we check for the case where no value will be provided where one is expected.
if output_def.runtime_type.is_nothing and not input_def.runtime_type.is_nothing:
raise DagsterInvalidDefinitionError((
'Input "{input_def.name}" to solid "{from_solid}" can not depend on the output '
'"{output_def.name}" from solid "{dep.solid}". '
'Input "{input_def.name}" expects a value of type '
'{input_def.runtime_type.display_name} and output "{output_def.name}" returns '
'type {output_def.runtime_type.display_name}{extra}.').format(
from_solid=from_solid,
dep=dep,
output_def=output_def,
input_def=input_def,
extra=' (which produces no value)'
if output_def.runtime_type.is_nothing else '',
))
def _validate_resource_dependencies(mode_definitions, solids):
'''This validation ensures that each pipeline context provides the resources that are required
by each solid.
'''
check.list_param(mode_definitions, 'mode_definintions', of_type=ModeDefinition)
check.list_param(solids, 'solids', of_type=ISolidDefinition)
for mode_def in mode_definitions:
mode_resources = set(mode_def.resource_defs.keys())
for solid in solids:
for resource in solid.resources:
if resource not in mode_resources:
raise DagsterInvalidDefinitionError(
(
'Resource "{resource}" is required by solid {solid_name}, but is not '
'provided by mode "{mode_name}"'
).format(resource=resource, solid_name=solid.name, mode_name=mode_def.name)
)
def __init__(self, inner_type: DagsterType):
inner_type = resolve_dagster_type(inner_type)
if inner_type is Nothing:
raise DagsterInvalidDefinitionError(
"Type Nothing can not be wrapped in List or Optional")
key = "Optional." + cast(str, inner_type.key)
self.inner_type = inner_type
super(OptionalType, self).__init__(
key=key,
name=None,
kind=DagsterTypeKind.NULLABLE,
type_check_fn=self.type_check_method,
loader=_create_nullable_input_schema(inner_type),
# This throws a type error with Py
typing_type=t.Optional[inner_type.typing_type], # type: ignore
)
def make_python_type_usable_as_dagster_type(python_type, dagster_type):
'''
Take any existing python type and map it to a dagster type (generally created with
:py:class:`DagsterType <dagster.DagsterType>`) This can only be called once
on a given python type.
'''
check.inst_param(dagster_type, 'dagster_type', DagsterType)
if python_type in _PYTHON_TYPE_TO_DAGSTER_TYPE_MAPPING_REGISTRY:
# This would be just a great place to insert a short URL pointing to the type system
# documentation into the error message
# https://github.com/dagster-io/dagster/issues/1831
raise DagsterInvalidDefinitionError((
'A Dagster type has already been registered for the Python type '
'{python_type}. make_python_type_usable_as_dagster_type can only '
'be called once on a python type as it is registering a 1:1 mapping '
'between that python type and a dagster type.').format(
python_type=python_type))
_PYTHON_TYPE_TO_DAGSTER_TYPE_MAPPING_REGISTRY[python_type] = dagster_type
def _validate_inputs(dependency_structure, solid_dict):
for solid in solid_dict.values():
for handle in solid.input_handles():
if not dependency_structure.has_deps(handle):
if (not handle.input_def.dagster_type.input_hydration_config
and not handle.input_def.dagster_type.kind
== DagsterTypeKind.NOTHING):
raise DagsterInvalidDefinitionError(
'Input "{input_name}" in solid "{solid_name}" is not connected to '
'the output of a previous solid and can not be hydrated from configuration, '
'creating an impossible to execute pipeline. '
'Possible solutions are:\n'
' * add a input_hydration_config for the type "{dagster_type}"\n'
' * connect "{input_name}" to the output of another solid\n'
.format(
solid_name=solid.name,
input_name=handle.input_def.name,
dagster_type=handle.input_def.dagster_type.
display_name,
))
def __new__(cls, dependencies):
from .composition import MappedInputPlaceholder
deps = check.list_param(dependencies, "dependencies")
seen = {}
for dep in deps:
if isinstance(dep, DependencyDefinition):
key = dep.solid + ":" + dep.output
if key in seen:
raise DagsterInvalidDefinitionError(
'Duplicate dependencies on solid "{dep.solid}" output "{dep.output}" '
"used in the same MultiDependencyDefinition.".format(dep=dep)
)
seen[key] = True
elif dep is MappedInputPlaceholder:
pass
else:
check.failed("Unexpected dependencies entry {}".format(dep))
return super(MultiDependencyDefinition, cls).__new__(cls, deps)
def _validate_input_output_pair(input_def, output_def, from_solid, dep):
# Currently, we opt to be overly permissive with input/output type mismatches.
# Here we check for the case where no value will be provided where one is expected.
if (output_def.dagster_type.kind == DagsterTypeKind.NOTHING
and not input_def.dagster_type.kind == DagsterTypeKind.NOTHING):
raise DagsterInvalidDefinitionError((
'Input "{input_def.name}" to node "{from_solid}" can not depend on the output '
'"{output_def.name}" from node "{dep.solid}". '
'Input "{input_def.name}" expects a value of type '
'{input_def.dagster_type.display_name} and output "{output_def.name}" returns '
"type {output_def.dagster_type.display_name}{extra}.").format(
from_solid=from_solid,
dep=dep,
output_def=output_def,
input_def=input_def,
extra=" (which produces no value)"
if output_def.dagster_type.kind == DagsterTypeKind.NOTHING else
"",
))
def _check_solid_defs(self):
solid_defs = {}
solid_to_pipeline = {}
for pipeline in self._all_pipelines:
for solid_def in pipeline.all_node_defs + [pipeline.graph]:
if solid_def.name not in solid_defs:
solid_defs[solid_def.name] = solid_def
solid_to_pipeline[solid_def.name] = pipeline.name
if not solid_defs[solid_def.name] is solid_def:
first_name, second_name = sorted(
[solid_to_pipeline[solid_def.name], pipeline.name]
)
raise DagsterInvalidDefinitionError(
(
f"Conflicting definitions found in repository with name '{solid_def.name}'. "
"Op/Graph/Solid definition names must be unique within a "
f"repository. {solid_def.__class__.__name__} is defined in {pipeline.target_type} "
f"'{first_name}' and in {pipeline.target_type} '{second_name}'."
)
)
def _infer_inputs_from_params(params, decorator_name, solid_name):
input_defs = []
for param in params:
try:
input_defs.append(InputDefinition(param.name, _input_param_type(param.annotation)))
except DagsterInvalidDefinitionError as type_error:
six.raise_from(
DagsterInvalidDefinitionError(
'Error inferring Dagster type for input name {param} typed as '
'"{type_annotation}" from {decorator} "{solid}". '
'Correct the issue or explicitly pass definitions to {decorator}.'.format(
decorator=decorator_name,
solid=solid_name,
param=param.name,
type_annotation=param.annotation,
)
),
type_error,
)
return input_defs
