def __init__(
self,
config,
default_value=FIELD_NO_DEFAULT_PROVIDED,
is_optional=None,
is_required=None,
description=None,
):
from .validate import validate_config
from .post_process import post_process_config
self.config_type = check.inst(self._resolve_config_arg(config), ConfigType)
self.description = check.opt_str_param(description, 'description')
check.opt_bool_param(is_optional, 'is_optional')
check.opt_bool_param(is_required, 'is_required')
canonical_is_required = canonicalize_backcompat_args(
new_val=is_required,
new_arg='is_required',
old_val=is_optional,
old_arg='is_optional',
coerce_old_to_new=lambda val: not val,
additional_warn_txt='"is_optional" deprecated in 0.7.0 and will be removed in 0.8.0. Users should use "is_required" instead.',
)
if canonical_is_required is True:
check.param_invariant(
default_value == FIELD_NO_DEFAULT_PROVIDED,
'default_value',
'required arguments should not specify default values',
)
self._default_value = default_value
# check explicit default value
if self.default_provided:
# invoke through property in case it is callable
value = self.default_value
evr = validate_config(self.config_type, value)
if not evr.success:
raise DagsterInvalidConfigError(
'Invalid default_value for Field.', evr.errors, default_value,
)
if canonical_is_required is None:
# neither is_required nor is_optional were specified
canonical_is_required = not all_optional_type(self.config_type)
# on implicitly optional - set the default value
# by resolving the defaults of the type
if not canonical_is_required and self._default_value == FIELD_NO_DEFAULT_PROVIDED:
evr = post_process_config(self.config_type, None)
if not evr.success:
raise DagsterInvalidConfigError(
'Unable to resolve implicit default_value for Field.', evr.errors, None,
)
self._default_value = evr.value
self.is_optional = not canonical_is_required
def build(pipeline_def, run_config=None, mode=None):
'''This method validates a given run config against the pipeline config schema. If
successful, we instantiate an EnvironmentConfig object.
In case the run_config is invalid, this method raises a DagsterInvalidConfigError
'''
from dagster.config.validate import process_config
from .composite_descent import composite_descent
check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
run_config = check.opt_dict_param(run_config, 'run_config')
check.opt_str_param(mode, 'mode')
mode = mode or pipeline_def.get_default_mode_name()
environment_type = create_environment_type(pipeline_def, mode)
config_evr = process_config(environment_type, run_config)
if not config_evr.success:
raise DagsterInvalidConfigError(
'Error in config for pipeline {}'.format(pipeline_def.name),
config_evr.errors,
run_config,
)
config_value = config_evr.value
mode_def = pipeline_def.get_mode_definition(mode)
resource_configs = config_value.get('resources', {})
processed_resource_configs = {}
for resource_key, resource_def in mode_def.resource_defs.items():
resource_config = resource_configs.get(resource_key, {})
resource_config_evr = resource_def.process_config(resource_config)
if not resource_config_evr.success:
raise DagsterInvalidConfigError(
'Error in config for resource {}'.format(resource_key),
resource_config_evr.errors,
resource_config,
)
else:
processed_resource_configs[
resource_key] = resource_config_evr.value
solid_config_dict = composite_descent(pipeline_def,
config_value.get('solids', {}))
# TODO: replace this with a simple call to from_dict of the config.get when ready to fully deprecate
temp_intermed = config_value.get('intermediate_storage')
if config_value.get('storage'):
if temp_intermed is None:
temp_intermed = {EmptyIntermediateStoreBackcompatConfig(): {}}
return EnvironmentConfig(
solids=solid_config_dict,
execution=ExecutionConfig.from_dict(config_value.get('execution')),
storage=StorageConfig.from_dict(config_value.get('storage')),
intermediate_storage=IntermediateStorageConfig.from_dict(
temp_intermed),
loggers=config_value.get('loggers'),
original_config_dict=run_config,
resources=processed_resource_configs,
)
def _resolve_bound_config(solid_config: Any,
solid_def: SolidDefinition) -> Any:
"""Validate config against config schema, and return validated config."""
from dagster.config.validate import process_config
# Config processing system expects the top level config schema to be a dictionary, but solid
# config schema can be scalar. Thus, we wrap it in another layer of indirection.
outer_config_shape = Shape({"config": solid_def.get_config_field()})
config_evr = process_config(
outer_config_shape, {"config": solid_config} if solid_config else {})
if not config_evr.success:
raise DagsterInvalidConfigError(
f"Error in config for {solid_def.node_type_str} ",
config_evr.errors,
solid_config,
)
validated_config = cast(Dict, config_evr.value).get("config")
mapped_config_evr = solid_def.apply_config_mapping(
{"config": validated_config})
if not mapped_config_evr.success:
raise DagsterInvalidConfigError(
f"Error in config for {solid_def.node_type_str} ",
mapped_config_evr.errors,
solid_config,
)
validated_config = cast(Dict, mapped_config_evr.value).get("config")
return validated_config
def __init__(
self, config, default_value=FIELD_NO_DEFAULT_PROVIDED, is_required=None, description=None,
):
from .validate import validate_config
from .post_process import resolve_defaults
self.config_type = check.inst(self._resolve_config_arg(config), ConfigType)
self.description = check.opt_str_param(description, "description")
check.opt_bool_param(is_required, "is_required")
if default_value != FIELD_NO_DEFAULT_PROVIDED:
check.param_invariant(
not (callable(default_value)), "default_value", "default_value cannot be a callable"
)
if is_required is True:
check.param_invariant(
default_value == FIELD_NO_DEFAULT_PROVIDED,
"default_value",
"required arguments should not specify default values",
)
self._default_value = default_value
# check explicit default value
if self.default_provided:
if self.config_type.kind == ConfigTypeKind.ENUM and is_enum_value(default_value):
raise DagsterInvalidDefinitionError(
(
"You have passed into a python enum value as the default value "
"into of a config enum type {name}. You must pass in the underlying "
"string represention as the default value. One of {value_set}."
).format(
value_set=[ev.config_value for ev in self.config_type.enum_values],
name=self.config_type.given_name,
)
)
evr = validate_config(self.config_type, default_value)
if not evr.success:
raise DagsterInvalidConfigError(
"Invalid default_value for Field.", evr.errors, default_value,
)
if is_required is None:
is_optional = has_implicit_default(self.config_type) or self.default_provided
is_required = not is_optional
# on implicitly optional - set the default value
# by resolving the defaults of the type
if is_optional and not self.default_provided:
evr = resolve_defaults(self.config_type, None)
if not evr.success:
raise DagsterInvalidConfigError(
"Unable to resolve implicit default_value for Field.", evr.errors, None,
)
self._default_value = evr.value
self._is_required = is_required
def dagster_instance_config(
base_dir,
config_filename=DAGSTER_CONFIG_YAML_FILENAME,
overrides=None,
):
check.str_param(base_dir, "base_dir")
check.invariant(os.path.isdir(base_dir), "base_dir should be a directory")
overrides = check.opt_dict_param(overrides, "overrides")
config_yaml_path = os.path.join(base_dir, config_filename)
if not os.path.exists(config_yaml_path):
warnings.warn((
"The dagster instance configuration file ({config_filename}) is not present at "
"{base_dir}. Dagster uses this file to know where and how to store "
"local artifacts, information about past runs, and structured events.\n"
"If nothing is specified, Dagster will store this information "
"in the local filesystem in the {base_dir} directory.").format(
config_filename=config_filename, base_dir=base_dir))
dagster_config_dict = merge_dicts(load_yaml_from_globs(config_yaml_path),
overrides)
if "custom_instance_class" in dagster_config_dict:
custom_instance_class_data = dagster_config_dict[
"custom_instance_class"]
validate_custom_config = validate_config(
configurable_class_schema(),
custom_instance_class_data,
)
if not validate_custom_config.success:
raise DagsterInvalidConfigError(
"Errors whilst loading dagster custom class config at {}".
format(config_filename),
validate_custom_config.errors,
custom_instance_class_data,
)
custom_instance_class = class_from_code_pointer(
custom_instance_class_data["module"],
custom_instance_class_data["class"])
schema = merge_dicts(dagster_instance_config_schema(),
custom_instance_class.config_schema())
else:
custom_instance_class = None
schema = dagster_instance_config_schema()
dagster_config = validate_config(schema, dagster_config_dict)
if not dagster_config.success:
raise DagsterInvalidConfigError(
"Errors whilst loading dagster instance config at {}.".format(
config_filename),
dagster_config.errors,
dagster_config_dict,
)
return (dagster_config.value, custom_instance_class)
def _check_invocation_requirements(
solid_def: "SolidDefinition", context: Optional["DirectSolidExecutionContext"]
) -> "DirectSolidExecutionContext":
"""Ensure that provided context fulfills requirements of solid definition.
If no context was provided, then construct an enpty DirectSolidExecutionContext
"""
from dagster.core.execution.context.invocation import DirectSolidExecutionContext
from dagster.config.validate import validate_config
# Check resource requirements
if solid_def.required_resource_keys and context is None:
raise DagsterInvalidInvocationError(
f'Solid "{solid_def.name}" has required resources, but no context was provided. Use the'
"`build_solid_context` function to construct a context with the required "
"resources."
)
if context is not None and solid_def.required_resource_keys:
resources_dict = cast( # type: ignore[attr-defined]
"DirectSolidExecutionContext",
context,
).resources._asdict()
for resource_key in solid_def.required_resource_keys:
if resource_key not in resources_dict:
raise DagsterInvalidInvocationError(
f'Solid "{solid_def.name}" requires resource "{resource_key}", but no resource '
"with that key was found on the context."
)
# Check config requirements
if not context and solid_def.config_schema.as_field().is_required:
raise DagsterInvalidInvocationError(
f'Solid "{solid_def.name}" has required config schema, but no context was provided. '
"Use the `build_solid_context` function to create a context with config."
)
config = None
if solid_def.config_field:
solid_config = check.opt_dict_param(
context.solid_config if context else None, "solid_config"
)
config_evr = validate_config(solid_def.config_field.config_type, solid_config)
if not config_evr.success:
raise DagsterInvalidConfigError(
"Error in config for solid ", config_evr.errors, solid_config
)
mapped_config_evr = solid_def.apply_config_mapping({"config": solid_config})
if not mapped_config_evr.success:
raise DagsterInvalidConfigError(
"Error in config for solid ", mapped_config_evr.errors, solid_config
)
config = mapped_config_evr.value.get("config", {})
return (
context
if context
else DirectSolidExecutionContext(solid_config=config, resources_dict=None, instance=None)
)
def dagster_instance_config(
base_dir,
config_filename=DAGSTER_CONFIG_YAML_FILENAME,
overrides=None,
):
check.str_param(base_dir, "base_dir")
check.invariant(os.path.isdir(base_dir), "base_dir should be a directory")
overrides = check.opt_dict_param(overrides, "overrides")
config_yaml_path = os.path.join(base_dir, config_filename)
if not os.path.exists(config_yaml_path) and is_dagster_home_set():
warnings.warn(
f"No dagster instance configuration file ({config_filename}) found at "
f"{base_dir}. Defaulting to loading and storing all metadata with {base_dir}. "
f"If this is the desired behavior, create an empty {config_filename} file in {base_dir}."
)
dagster_config_dict = merge_dicts(load_yaml_from_globs(config_yaml_path),
overrides)
if "instance_class" in dagster_config_dict:
custom_instance_class_data = dagster_config_dict["instance_class"]
validate_custom_config = validate_config(
configurable_class_schema(),
custom_instance_class_data,
)
if not validate_custom_config.success:
raise DagsterInvalidConfigError(
"Errors whilst loading dagster custom class config at {}".
format(config_filename),
validate_custom_config.errors,
custom_instance_class_data,
)
custom_instance_class = class_from_code_pointer(
custom_instance_class_data["module"],
custom_instance_class_data["class"])
schema = merge_dicts(dagster_instance_config_schema(),
custom_instance_class.config_schema())
else:
custom_instance_class = None
schema = dagster_instance_config_schema()
dagster_config = validate_config(schema, dagster_config_dict)
if not dagster_config.success:
raise DagsterInvalidConfigError(
"Errors whilst loading dagster instance config at {}.".format(
config_filename),
dagster_config.errors,
dagster_config_dict,
)
return (dagster_config.value, custom_instance_class)
def resolve_from_unvalidated_config(self, config: Any) -> Any:
"""Validates config against outer config schema, and calls mapping against validated config."""
receive_processed_config_values = check.opt_bool_param(
self.receive_processed_config_values,
"receive_processed_config_values",
default=True)
if receive_processed_config_values:
outer_evr = process_config(
self.config_schema.config_type,
config,
)
else:
outer_evr = validate_config(
self.config_schema.config_type,
config,
)
if not outer_evr.success:
raise DagsterInvalidConfigError(
"Error in config mapping ",
outer_evr.errors,
config,
)
outer_config = outer_evr.value
if not receive_processed_config_values:
outer_config = resolve_defaults(
cast(ConfigType, self.config_schema.config_type),
outer_config,
).value
return self.config_fn(outer_config)
def config_map_resources(
resource_defs: Dict[str, ResourceDefinition],
resource_configs: Dict[str, Any],
) -> Dict[str, ResourceConfig]:
"""This function executes the config mappings for resources with respect to ConfigurableDefinition.
It iterates over resource_defs and looks up the corresponding config because resources need to
be mapped regardless of whether they receive config from run_config."""
config_mapped_resource_configs = {}
for resource_key, resource_def in resource_defs.items():
resource_config = resource_configs.get(resource_key, {})
resource_config_evr = resource_def.apply_config_mapping(
resource_config)
if not resource_config_evr.success:
raise DagsterInvalidConfigError(
"Error in config for resource {}".format(resource_key),
resource_config_evr.errors,
resource_config,
)
else:
config_mapped_resource_configs[
resource_key] = ResourceConfig.from_dict(
resource_config_evr.value)
return config_mapped_resource_configs
def config_map_objects(config_value, defs, keyed_by, def_type, name_of_def_type):
"""This function executes the config mappings for executors and {system, intermediate} storage
definitions with respect to ConfigurableDefinition. It calls the ensure_single_item macro on the
incoming config and then applies config mapping to the result and the first executor_def with
the same name on the mode_def."""
config = config_value.get(keyed_by)
check.opt_dict_param(config, "config", key_type=str)
if not config:
return None
obj_name, obj_config = ensure_single_item(config)
obj_def = next(
(defi for defi in defs if defi.name == obj_name), None
) # obj_defs are stored in a list and we want to find the def matching name
check.inst(
obj_def,
def_type,
(
"Could not find a {def_type} definition on the selected mode that matches the "
'{def_type} "{obj_name}" given in run config'
).format(def_type=def_type, obj_name=obj_name),
)
obj_config_evr = obj_def.apply_config_mapping(obj_config)
if not obj_config_evr.success:
raise DagsterInvalidConfigError(
'Invalid configuration provided for {} "{}"'.format(name_of_def_type, obj_name),
obj_config_evr.errors,
obj_config,
)
return {obj_name: obj_config_evr.value}
def raise_composite_descent_config_error(descent_stack, failed_config_value,
evr):
check.inst_param(descent_stack, 'descent_stack', DescentStack)
check.inst_param(evr, 'evr', EvaluateValueResult)
solid = descent_stack.current_solid
message = 'In pipeline {pipeline_name} at stack {stack}: \n'.format(
pipeline_name=descent_stack.pipeline_def.name,
stack=':'.join(descent_stack.handle.path),
)
message += (
'Solid "{solid_name}" with definition "{solid_def_name}" has a '
'configuration error. '
'It has produced config a via its config_fn that fails to '
'pass validation in the solids that it contains. '
'This indicates an error in the config mapping function itself. It must '
'produce correct config for its constiuent solids in all cases. The correct '
'resolution is to fix the mapping function. Details on the error (and the paths '
'on this error are relative to config mapping function "root", not the entire document): '
).format(
solid_name=solid.name,
solid_def_name=solid.definition.name,
)
raise DagsterInvalidConfigError(message, evr.errors, failed_config_value)
def _get_host_mode_executor(recon_pipeline, run_config, get_executor_def_fn,
instance):
execution_config = run_config.get("execution")
if execution_config:
executor_name, executor_config = ensure_single_item(execution_config)
else:
executor_name = None
executor_config = {}
executor_def = get_executor_def_fn(executor_name)
executor_config_type = def_config_field(executor_def).config_type
config_evr = process_config(executor_config_type, executor_config)
if not config_evr.success:
raise DagsterInvalidConfigError(
"Error in executor config for executor {}".format(
executor_def.name),
config_evr.errors,
executor_config,
)
executor_config_value = config_evr.value
init_context = InitExecutorContext(
pipeline=recon_pipeline,
executor_def=executor_def,
executor_config=executor_config_value["config"],
instance=instance,
)
check_cross_process_constraints(init_context)
return executor_def.executor_creation_fn(init_context)
def rehydrate(self):
from dagster.config.field import resolve_to_config_type
from dagster.config.validate import process_config
from dagster.core.errors import DagsterInvalidConfigError
try:
module = importlib.import_module(self.module_name)
except ModuleNotFoundError:
check.failed(
f"Couldn't import module {self.module_name} when attempting to load the "
f"configurable class {self.module_name}.{self.class_name}")
try:
klass = getattr(module, self.class_name)
except AttributeError:
check.failed(
f"Couldn't find class {self.class_name} in module when attempting to load the "
f"configurable class {self.module_name}.{self.class_name}")
if not issubclass(klass, ConfigurableClass):
raise check.CheckError(
klass,
f"class {self.class_name} in module {self.module_name}",
ConfigurableClass,
)
config_dict = self.config_dict
result = process_config(resolve_to_config_type(klass.config_type()),
config_dict)
if not result.success:
raise DagsterInvalidConfigError(
f"Errors whilst loading configuration for {klass.config_type()}.",
result.errors,
config_dict,
)
return klass.from_config_value(self, result.value)
def build(pipeline, environment_dict=None, run_config=None):
from dagster.core.types.config.evaluator.composite_descent import composite_descent
from dagster.core.types.config.evaluator.validate import process_config
check.inst_param(pipeline, 'pipeline', PipelineDefinition)
environment_dict = check.opt_dict_param(environment_dict, 'environment_dict')
run_config = check.opt_inst_param(run_config, 'run_config', IRunConfig, default=RunConfig())
mode = run_config.mode or pipeline.get_default_mode_name()
environment_type = create_environment_type(pipeline, mode)
config_evr = process_config(environment_type, environment_dict)
if not config_evr.success:
raise DagsterInvalidConfigError(
'Error in config for pipeline {}'.format(pipeline.name),
config_evr.errors,
environment_dict,
)
config_value = config_evr.value
solid_config_dict = composite_descent(pipeline, config_value.get('solids', {}), run_config)
return EnvironmentConfig(
solids=solid_config_dict,
execution=ExecutionConfig.from_dict(config_value.get('execution')),
storage=StorageConfig.from_dict(config_value.get('storage')),
loggers=config_value.get('loggers'),
original_config_dict=environment_dict,
resources=config_value.get('resources'),
)
def create_from_config(run_container_context):
run_docker_container_context = (run_container_context.get(
"docker", {}) if run_container_context else {})
if not run_docker_container_context:
return DockerContainerContext()
processed_container_context = process_config(
DOCKER_CONTAINER_CONTEXT_SCHEMA, run_docker_container_context)
if not processed_container_context.success:
raise DagsterInvalidConfigError(
"Errors while parsing Docker container context",
processed_container_context.errors,
run_docker_container_context,
)
processed_context_value = processed_container_context.value
return DockerContainerContext(
registry=processed_context_value.get("registry"),
env_vars=processed_context_value.get("env_vars", []),
networks=processed_context_value.get("networks", []),
container_kwargs=processed_context_value.get("container_kwargs"),
)
def _get_host_mode_executor(recon_pipeline, run_config, executor_defs,
instance):
execution_config = run_config.get("execution", {})
execution_config_type = Field(selector_for_named_defs(executor_defs),
default_value={
executor_defs[0].name: {}
}).config_type
config_evr = process_config(execution_config_type, execution_config)
if not config_evr.success:
raise DagsterInvalidConfigError(
"Error processing execution config {}".format(execution_config),
config_evr.errors,
execution_config,
)
execution_config_value = config_evr.value
executor_name, executor_config = ensure_single_item(execution_config_value)
executor_defs_by_name = {
executor_def.name: executor_def
for executor_def in executor_defs
}
executor_def = executor_defs_by_name[executor_name]
init_context = InitExecutorContext(
job=recon_pipeline,
executor_def=executor_def,
executor_config=executor_config["config"],
instance=instance,
)
check_cross_process_constraints(init_context)
return executor_def.executor_creation_fn(init_context)
def create_from_config(run_container_context) -> "K8sContainerContext":
run_k8s_container_context = (run_container_context.get("k8s", {})
if run_container_context else {})
if not run_k8s_container_context:
return K8sContainerContext()
processed_container_context = process_config(
DagsterK8sJobConfig.config_type_container_context(),
run_k8s_container_context)
if not processed_container_context.success:
raise DagsterInvalidConfigError(
"Errors while parsing k8s container context",
processed_container_context.errors,
run_k8s_container_context,
)
processed_context_value = processed_container_context.value
return K8sContainerContext(
image_pull_policy=processed_context_value.get("image_pull_policy"),
image_pull_secrets=processed_context_value.get(
"image_pull_secrets"),
service_account_name=processed_context_value.get(
"service_account_name"),
env_config_maps=processed_context_value.get("env_config_maps"),
env_secrets=processed_context_value.get("env_secrets"),
env_vars=processed_context_value.get("env_vars"),
volume_mounts=processed_context_value.get("volume_mounts"),
volumes=processed_context_value.get("volumes"),
labels=processed_context_value.get("labels"),
namespace=processed_context_value.get("namespace"),
)
def config_map_executors(pipeline_def, config_value, mode):
'''This function executes the config mappings for executors with respect to IConfigMappable.
It calls the ensure_single_item macro on the incoming config and then applies config mapping to
the result and the first executor_def with the same name on the mode_def.'''
config = config_value.get('execution')
check.opt_dict_param(config, 'config', key_type=str)
if not config:
return None
executor_name, executor_config = ensure_single_item(config)
mode_def = pipeline_def.get_mode_definition(mode)
executor_def = next(
(defi
for defi in mode_def.executor_defs if defi.name == executor_name),
None
) # executor_defs are stored in a list and we want to find the def matching name
check.inst(executor_def, ExecutorDefinition)
executor_config_evr = executor_def.apply_config_mapping(executor_config)
if not executor_config_evr.success:
raise DagsterInvalidConfigError(
'Error in config for executor {}'.format(executor_name),
executor_config_evr.errors,
executor_config,
)
return {executor_name: executor_config_evr.value}
def get_user_defined_k8s_config(tags):
check.inst_param(tags, "tags", frozentags)
if not any(key in tags for key in [K8S_RESOURCE_REQUIREMENTS_KEY, USER_DEFINED_K8S_CONFIG_KEY]):
return UserDefinedDagsterK8sConfig()
user_defined_k8s_config = {}
if USER_DEFINED_K8S_CONFIG_KEY in tags:
user_defined_k8s_config_value = json.loads(tags[USER_DEFINED_K8S_CONFIG_KEY])
result = validate_config(USER_DEFINED_K8S_CONFIG_SCHEMA, user_defined_k8s_config_value)
if not result.success:
raise DagsterInvalidConfigError(
"Error in tags for {}".format(USER_DEFINED_K8S_CONFIG_KEY), result.errors, result,
)
user_defined_k8s_config = result.value
container_config = user_defined_k8s_config.get("container_config", {})
# Backcompat for resource requirements key
if K8S_RESOURCE_REQUIREMENTS_KEY in tags:
resource_requirements_config = get_k8s_resource_requirements(tags)
container_config = merge_dicts(
container_config, {"resources": resource_requirements_config}
)
return UserDefinedDagsterK8sConfig(
container_config=container_config,
pod_template_spec_metadata=user_defined_k8s_config.get("pod_template_spec_metadata"),
pod_spec_config=user_defined_k8s_config.get("pod_spec_config"),
job_config=user_defined_k8s_config.get("job_config"),
job_spec_config=user_defined_k8s_config.get("job_spec_config"),
)
def _composite_descent(parent_stack, solids_config_dict, resource_defs):
"""
The core implementation of composite_descent. This yields a stream of
SolidConfigEntry. This is used by composite_descent to construct a
dictionary.
It descends over the entire solid hierarchy, constructing an entry
for every handle. If it encounters a composite solid instance
with a config mapping, it will invoke that config mapping fn,
producing the config that is necessary to configure the child solids.
This process unrolls recursively as you descend down the tree.
"""
for solid in parent_stack.current_container.solids:
current_stack = parent_stack.descend(solid)
current_handle = current_stack.handle
current_solid_config = solids_config_dict.get(solid.name, {})
# the base case
if isinstance(solid.definition, SolidDefinition):
config_mapped_solid_config = solid.definition.apply_config_mapping(
{"config": current_solid_config.get("config")})
if not config_mapped_solid_config.success:
raise DagsterInvalidConfigError(
"Error in config for solid {}".format(solid.name),
config_mapped_solid_config.errors,
config_mapped_solid_config,
)
complete_config_object = merge_dicts(
current_solid_config, config_mapped_solid_config.value)
yield SolidConfigEntry(
current_handle, SolidConfig.from_dict(complete_config_object))
continue
graph_def = check.inst(solid.definition, GraphDefinition)
yield SolidConfigEntry(
current_handle,
SolidConfig.from_dict({
"inputs":
current_solid_config.get("inputs"),
"outputs":
current_solid_config.get("outputs"),
}),
)
# If there is a config mapping, invoke it and get the descendent solids
# config that way. Else just grabs the solids entry of the current config
solids_dict = (_get_mapped_solids_dict(
solid, graph_def, current_stack, current_solid_config,
resource_defs) if graph_def.config_mapping else
current_solid_config.get("solids", {}))
yield from _composite_descent(current_stack, solids_dict,
resource_defs)
def _composite_descent(parent_stack, solids_config_dict):
'''
The core implementation of composite_descent. This yields a stream of
SolidConfigEntry. This is used by composite_descent to construct a
dictionary.
It descends over the entire solid hierarchy, constructing an entry
for every handle. If it encounters a composite solid instance
with a config mapping, it will invoke that config mapping fn,
producing the config that is necessary to configure the child solids.
This process unrolls recursively as you descend down the tree.
'''
for solid in parent_stack.current_container.solids:
current_stack = parent_stack.descend(solid)
current_handle = current_stack.handle
current_solid_config = solids_config_dict.get(solid.name, {})
# the base case
if not isinstance(solid.definition, CompositeSolidDefinition):
transformed_current_solid_config = solid.definition.apply_config_mapping(
current_solid_config)
if not transformed_current_solid_config.success:
raise DagsterInvalidConfigError(
'Error in config for solid {}'.format(solid.name),
transformed_current_solid_config.errors,
transformed_current_solid_config,
)
yield SolidConfigEntry(
current_handle,
SolidConfig.from_dict(transformed_current_solid_config.value))
continue
composite_def = check.inst(solid.definition, CompositeSolidDefinition)
yield SolidConfigEntry(
current_handle,
SolidConfig.from_dict({
'inputs':
current_solid_config.get('inputs'),
'outputs':
current_solid_config.get('outputs'),
}),
)
# If there is a config mapping, invoke it and get the descendent solids
# config that way. Else just grabs the solids entry of the current config
solids_dict = (_get_mapped_solids_dict(composite_def, current_stack,
current_solid_config)
if composite_def.config_mapping else
current_solid_config.get('solids', {}))
for sce in _composite_descent(current_stack, solids_dict):
yield sce
def dagster_instance_config(base_dir):
dagster_config_dict = load_yaml_from_globs(
os.path.join(base_dir, DAGSTER_CONFIG_YAML_FILENAME))
dagster_config_type = define_dagster_config_cls().inst()
dagster_config = evaluate_config(dagster_config_type, dagster_config_dict)
if not dagster_config.success:
raise DagsterInvalidConfigError(None, dagster_config.errors,
dagster_config_dict)
return dagster_config.value
def _get_mapped_solids_dict(
composite,
graph_def,
current_stack,
current_solid_config,
resource_defs,
is_using_graph_job_op_apis,
):
# the spec of the config mapping function is that it takes the dictionary at:
# solid_name:
# config: {dict_passed_to_user}
# and it returns the dictionary rooted at solids
# solid_name:
# solids: {return_value_of_config_fn}
# We must call the config mapping function and then validate it against
# the child schema.
# apply @configured config mapping to the composite's incoming config before we get to the
# composite's own config mapping process
config_mapped_solid_config = graph_def.apply_config_mapping(
current_solid_config)
if not config_mapped_solid_config.success:
raise DagsterInvalidConfigError(
"Error in config for composite solid {}".format(composite.name),
config_mapped_solid_config.errors,
config_mapped_solid_config,
)
with user_code_error_boundary(DagsterConfigMappingFunctionError,
_get_error_lambda(current_stack)):
mapped_solids_config = graph_def.config_mapping.resolve_from_validated_config(
config_mapped_solid_config.value.get("config", {}))
# Dynamically construct the type that the output of the config mapping function will
# be evaluated against
type_to_evaluate_against = define_solid_dictionary_cls(
solids=graph_def.solids,
ignored_solids=None,
dependency_structure=graph_def.dependency_structure,
parent_handle=current_stack.handle,
resource_defs=resource_defs,
is_using_graph_job_op_apis=is_using_graph_job_op_apis,
)
# process against that new type
evr = process_config(type_to_evaluate_against, mapped_solids_config)
if not evr.success:
raise_composite_descent_config_error(current_stack,
mapped_solids_config, evr)
return evr.value
def dagster_instance_config(base_dir, config_filename=DAGSTER_CONFIG_YAML_FILENAME, overrides=None):
overrides = check.opt_dict_param(overrides, 'overrides')
dagster_config_dict = merge_dicts(
load_yaml_from_globs(os.path.join(base_dir, config_filename)), overrides
)
dagster_config_type = define_dagster_config_cls().inst()
dagster_config = evaluate_config(dagster_config_type, dagster_config_dict)
if not dagster_config.success:
raise DagsterInvalidConfigError(None, dagster_config.errors, dagster_config_dict)
return dagster_config.value
def ensure_workspace_config(workspace_config, yaml_path):
check.dict_param(workspace_config, 'workspace_config')
check.str_param(yaml_path, 'yaml_path')
validation_result = process_workspace_config(workspace_config)
if not validation_result.success:
raise DagsterInvalidConfigError(
'Errors while loading workspace config at {}.'.format(yaml_path),
validation_result.errors,
workspace_config,
)
return validation_result
def get_validated_config(config_yaml=None):
config_type = celery_executor.config_schema.config_type
config_value = get_config_value_from_yaml(config_yaml)
config = validate_config(config_type, config_value)
if not config.success:
raise DagsterInvalidConfigError(
"Errors while loading Celery executor config at {}.".format(
config_yaml),
config.errors,
config_value,
)
return post_process_config(config_type, config_value).value
def ensure_workspace_config(workspace_config, yaml_path):
check.str_param(yaml_path, "yaml_path")
check.dict_param(workspace_config, "workspace_config")
validation_result = process_workspace_config(workspace_config)
if not validation_result.success:
raise DagsterInvalidConfigError(
"Errors while loading workspace config from {yaml_path}.".format(
yaml_path=os.path.abspath(yaml_path)),
validation_result.errors,
workspace_config,
)
return validation_result
def _get_mapped_resource_config(
resource_defs: Dict[str, ResourceDefinition],
run_config: Dict[str, Any]) -> Dict[str, ResourceConfig]:
resource_config_schema = define_resource_dictionary_cls(resource_defs)
config_evr = process_config(resource_config_schema, run_config)
if not config_evr.success:
raise DagsterInvalidConfigError(
"Error in config for resources ",
config_evr.errors,
run_config,
)
config_value = config_evr.value
return config_map_resources(resource_defs, config_value)
def config_map_executor(
executor_config: Dict[str, Any],
executor_def: ExecutorDefinition,
) -> Dict[str, Any]:
executor_config_evr = executor_def.apply_config_mapping(executor_config)
if not executor_config_evr.success:
raise DagsterInvalidConfigError(
f"Invalid configuration provided for executor '{executor_def.name}'",
executor_config_evr.errors,
executor_config,
)
return {executor_def.name: executor_config_evr.value}
def get_k8s_resource_requirements(tags):
check.inst_param(tags, "tags", frozentags)
check.invariant(K8S_RESOURCE_REQUIREMENTS_KEY in tags)
resource_requirements = json.loads(tags[K8S_RESOURCE_REQUIREMENTS_KEY])
result = validate_config(K8S_RESOURCE_REQUIREMENTS_SCHEMA, resource_requirements)
if not result.success:
raise DagsterInvalidConfigError(
"Error in tags for {}".format(K8S_RESOURCE_REQUIREMENTS_KEY), result.errors, result,
)
return result.value