def from_dict(config=None):
check.opt_dict_param(config, "config", key_type=str)
if config:
execution_engine_name, execution_engine_config = ensure_single_item(
config)
return ExecutionConfig(execution_engine_name,
execution_engine_config.get("config"))
return ExecutionConfig(None, None)
def from_dict(config=None):
check.opt_dict_param(config, "config", key_type=str)
if config:
system_storage_name, system_storage_config = ensure_single_item(
config)
return StorageConfig(system_storage_name,
system_storage_config.get("config"))
return StorageConfig(None, None)
def from_dict(config=None):
check.opt_dict_param(config, "config", key_type=str)
if config:
intermediate_storage_name, intermediate_storage_config = ensure_single_item(
config)
return IntermediateStorageConfig(
intermediate_storage_name,
intermediate_storage_config.get("config"))
return IntermediateStorageConfig(None, None)
def post_process_config_to_selector(selector_type, config_value):
check.param_invariant(
selector_type.kind == ConfigTypeKind.SELECTOR,
'selector_type',
'Non-selector not caught in validation',
)
if config_value:
check.invariant(config_value and len(config_value) == 1)
field_name, incoming_field_value = ensure_single_item(config_value)
else:
field_name, field_def = ensure_single_item(selector_type.fields)
incoming_field_value = field_def.default_value if field_def.default_provided else None
parent_field = selector_type.fields[field_name]
field_value = post_process_config(parent_field.config_type,
incoming_field_value)
return {field_name: field_value}
def validate_selector_config(
context: ValidationContext, config_value: object
) -> EvaluateValueResult[Dict[str, object]]:
check.inst_param(context, "context", ValidationContext)
check.param_invariant(context.config_type_snap.kind == ConfigTypeKind.SELECTOR, "selector_type")
check.not_none_param(config_value, "config_value")
# Special case the empty dictionary, meaning no values provided for the
# value of the selector. # E.g. {'logging': {}}
# If there is a single field defined on the selector and if it is optional
# it passes validation. (e.g. a single logger "console")
if config_value == {}:
return _validate_empty_selector_config(context) # type: ignore
# Now we ensure that the used-provided config has only a a single entry
# and then continue the validation pass
if not isinstance(config_value, dict):
return EvaluateValueResult.for_error(create_selector_type_error(context, config_value))
if len(config_value) > 1:
return EvaluateValueResult.for_error(
create_selector_multiple_fields_error(context, config_value)
)
field_name, field_value = ensure_single_item(config_value)
if not context.config_type_snap.has_field(field_name):
return EvaluateValueResult.for_error(create_field_not_defined_error(context, field_name))
field_snap = context.config_type_snap.get_field(field_name)
child_evaluate_value_result = _validate_config(
context.for_field_snap(field_snap),
# This is a very particular special case where we want someone
# to be able to select a selector key *without* a value
#
# e.g.
# storage:
# filesystem:
#
# And we want the default values of the child elements of filesystem:
# to "fill in"
{}
if field_value is None
and ConfigTypeKind.has_fields(
context.config_schema_snapshot.get_config_snap(field_snap.type_key).kind
)
else field_value,
)
if child_evaluate_value_result.success:
return EvaluateValueResult.for_value( # type: ignore
frozendict({field_name: child_evaluate_value_result.value})
)
else:
return child_evaluate_value_result # type: ignore
def evaluate_selector_config(context):
check.inst_param(context, 'context', TraversalContext)
check.param_invariant(context.config_type.is_selector, 'selector_type')
if context.config_value:
if not isinstance(context.config_value, dict):
return EvaluateValueResult.for_error(
create_selector_type_error(context))
if len(context.config_value) > 1:
return EvaluateValueResult.for_error(
create_selector_multiple_fields_error(context))
field_name, incoming_field_value = ensure_single_item(
context.config_value)
if field_name not in context.config_type.fields:
return EvaluateValueResult.for_error(
create_field_not_defined_error(context, field_name))
else:
if len(context.config_type.fields) > 1:
return EvaluateValueResult.for_error(
create_selector_multiple_fields_no_field_selected_error(
context))
field_name, field_def = ensure_single_item(context.config_type.fields)
if not field_def.is_optional:
return EvaluateValueResult.for_error(
create_selector_unspecified_value_error(context))
incoming_field_value = field_def.default_value if field_def.default_provided else None
field_def = context.config_type.fields[field_name]
child_evaluate_value_result = _evaluate_config(
context.for_field(field_def, field_name, incoming_field_value))
if child_evaluate_value_result.success:
return EvaluateValueResult.for_value(
{field_name: child_evaluate_value_result.value})
else:
return child_evaluate_value_result
def from_dict(config=None):
check.opt_dict_param(config,
'config',
key_type=(str,
EmptyIntermediateStoreBackcompatConfig))
if config:
intermediate_storage_name, intermediate_storage_config = ensure_single_item(
config)
return IntermediateStorageConfig(
intermediate_storage_name,
intermediate_storage_config.get('config'))
return IntermediateStorageConfig(None, None)
def _validate_empty_selector_config(context):
if len(context.config_type.fields) > 1:
return EvaluateValueResult.for_error(
create_selector_multiple_fields_no_field_selected_error(context)
)
_, defined_field_def = ensure_single_item(context.config_type.fields)
if not defined_field_def.is_optional:
return EvaluateValueResult.for_error(create_selector_unspecified_value_error(context))
return EvaluateValueResult.for_value({})
def from_dict(config=None):
check.opt_dict_param(config, "config", key_type=str)
if config:
warnings.warn((
'The "storage" and "intermediate_storage" entries in the run config are deprecated, '
"and will removed in 0.11.0. Loading inputs and storing outputs are now handled "
'by "object managers", which are resources that can be configured via '
'the "object_manager" resource key.'))
intermediate_storage_name, intermediate_storage_config = ensure_single_item(
config)
return IntermediateStorageConfig(
intermediate_storage_name,
intermediate_storage_config.get("config"))
return IntermediateStorageConfig(None, None)
def config_map_objects(
config_value: Any,
defs: List[ExecutorDefinition],
keyed_by: str,
def_type: Type,
name_of_def_type: str,
) -> Optional[Dict[str, Any]]:
"""This function executes the config mappings for executors definitions with respect to
ConfigurableDefinition. It calls the ensure_single_item macro on the incoming config and then
applies config mapping to the result and the first executor_def with the same name on
the mode_def."""
config = config_value.get(keyed_by)
check.opt_dict_param(config, "config", key_type=str)
if not config:
return None
obj_name, obj_config = ensure_single_item(config)
obj_def = next(
(defi for defi in defs if defi.name == obj_name), None
) # obj_defs are stored in a list and we want to find the def matching name
check.inst(
obj_def,
def_type,
("Could not find a {def_type} definition on the selected mode that matches the "
'{def_type} "{obj_name}" given in run config').format(
def_type=def_type, obj_name=obj_name),
)
obj_def = cast(ConfigurableDefinition, obj_def)
obj_config_evr = obj_def.apply_config_mapping(obj_config)
if not obj_config_evr.success:
raise DagsterInvalidConfigError(
'Invalid configuration provided for {} "{}"'.format(
name_of_def_type, obj_name),
obj_config_evr.errors,
obj_config,
)
return {obj_name: obj_config_evr.value}
def _selector(context, config_value, runtime_value):
selector_key, selector_value = ensure_single_item(config_value)
return func(context, selector_key, selector_value, runtime_value)
def test_ensure_single_item():
assert ensure_single_item({"foo": "bar"}) == ("foo", "bar")
with pytest.raises(ParameterCheckError, match="Expected dict with single item"):
ensure_single_item({"foo": "bar", "baz": "quux"})
def test_ensure_single_item():
assert ensure_single_item({'foo': 'bar'}) == ('foo', 'bar')
with pytest.raises(ParameterCheckError,
match='Expected dict with single item'):
ensure_single_item({'foo': 'bar', 'baz': 'quux'})
def _is_selector_field_optional(config_type: Selector) -> bool:
if len(config_type.fields) > 1:
return False
else:
_name, field = ensure_single_item(config_type.fields)
return not field.is_required
