def from_dict(config=None): check.opt_dict_param(config, "config", key_type=str) if config: execution_engine_name, execution_engine_config = ensure_single_item( config) return ExecutionConfig(execution_engine_name, execution_engine_config.get("config")) return ExecutionConfig(None, None)
def from_dict(config=None): check.opt_dict_param(config, "config", key_type=str) if config: system_storage_name, system_storage_config = ensure_single_item( config) return StorageConfig(system_storage_name, system_storage_config.get("config")) return StorageConfig(None, None)
def from_dict(config=None): check.opt_dict_param(config, "config", key_type=str) if config: intermediate_storage_name, intermediate_storage_config = ensure_single_item( config) return IntermediateStorageConfig( intermediate_storage_name, intermediate_storage_config.get("config")) return IntermediateStorageConfig(None, None)
def post_process_config_to_selector(selector_type, config_value): check.param_invariant( selector_type.kind == ConfigTypeKind.SELECTOR, 'selector_type', 'Non-selector not caught in validation', ) if config_value: check.invariant(config_value and len(config_value) == 1) field_name, incoming_field_value = ensure_single_item(config_value) else: field_name, field_def = ensure_single_item(selector_type.fields) incoming_field_value = field_def.default_value if field_def.default_provided else None parent_field = selector_type.fields[field_name] field_value = post_process_config(parent_field.config_type, incoming_field_value) return {field_name: field_value}
def validate_selector_config( context: ValidationContext, config_value: object ) -> EvaluateValueResult[Dict[str, object]]: check.inst_param(context, "context", ValidationContext) check.param_invariant(context.config_type_snap.kind == ConfigTypeKind.SELECTOR, "selector_type") check.not_none_param(config_value, "config_value") # Special case the empty dictionary, meaning no values provided for the # value of the selector. # E.g. {'logging': {}} # If there is a single field defined on the selector and if it is optional # it passes validation. (e.g. a single logger "console") if config_value == {}: return _validate_empty_selector_config(context) # type: ignore # Now we ensure that the used-provided config has only a a single entry # and then continue the validation pass if not isinstance(config_value, dict): return EvaluateValueResult.for_error(create_selector_type_error(context, config_value)) if len(config_value) > 1: return EvaluateValueResult.for_error( create_selector_multiple_fields_error(context, config_value) ) field_name, field_value = ensure_single_item(config_value) if not context.config_type_snap.has_field(field_name): return EvaluateValueResult.for_error(create_field_not_defined_error(context, field_name)) field_snap = context.config_type_snap.get_field(field_name) child_evaluate_value_result = _validate_config( context.for_field_snap(field_snap), # This is a very particular special case where we want someone # to be able to select a selector key *without* a value # # e.g. # storage: # filesystem: # # And we want the default values of the child elements of filesystem: # to "fill in" {} if field_value is None and ConfigTypeKind.has_fields( context.config_schema_snapshot.get_config_snap(field_snap.type_key).kind ) else field_value, ) if child_evaluate_value_result.success: return EvaluateValueResult.for_value( # type: ignore frozendict({field_name: child_evaluate_value_result.value}) ) else: return child_evaluate_value_result # type: ignore
def evaluate_selector_config(context): check.inst_param(context, 'context', TraversalContext) check.param_invariant(context.config_type.is_selector, 'selector_type') if context.config_value: if not isinstance(context.config_value, dict): return EvaluateValueResult.for_error( create_selector_type_error(context)) if len(context.config_value) > 1: return EvaluateValueResult.for_error( create_selector_multiple_fields_error(context)) field_name, incoming_field_value = ensure_single_item( context.config_value) if field_name not in context.config_type.fields: return EvaluateValueResult.for_error( create_field_not_defined_error(context, field_name)) else: if len(context.config_type.fields) > 1: return EvaluateValueResult.for_error( create_selector_multiple_fields_no_field_selected_error( context)) field_name, field_def = ensure_single_item(context.config_type.fields) if not field_def.is_optional: return EvaluateValueResult.for_error( create_selector_unspecified_value_error(context)) incoming_field_value = field_def.default_value if field_def.default_provided else None field_def = context.config_type.fields[field_name] child_evaluate_value_result = _evaluate_config( context.for_field(field_def, field_name, incoming_field_value)) if child_evaluate_value_result.success: return EvaluateValueResult.for_value( {field_name: child_evaluate_value_result.value}) else: return child_evaluate_value_result
def from_dict(config=None): check.opt_dict_param(config, 'config', key_type=(str, EmptyIntermediateStoreBackcompatConfig)) if config: intermediate_storage_name, intermediate_storage_config = ensure_single_item( config) return IntermediateStorageConfig( intermediate_storage_name, intermediate_storage_config.get('config')) return IntermediateStorageConfig(None, None)
def _validate_empty_selector_config(context): if len(context.config_type.fields) > 1: return EvaluateValueResult.for_error( create_selector_multiple_fields_no_field_selected_error(context) ) _, defined_field_def = ensure_single_item(context.config_type.fields) if not defined_field_def.is_optional: return EvaluateValueResult.for_error(create_selector_unspecified_value_error(context)) return EvaluateValueResult.for_value({})
def from_dict(config=None): check.opt_dict_param(config, "config", key_type=str) if config: warnings.warn(( 'The "storage" and "intermediate_storage" entries in the run config are deprecated, ' "and will removed in 0.11.0. Loading inputs and storing outputs are now handled " 'by "object managers", which are resources that can be configured via ' 'the "object_manager" resource key.')) intermediate_storage_name, intermediate_storage_config = ensure_single_item( config) return IntermediateStorageConfig( intermediate_storage_name, intermediate_storage_config.get("config")) return IntermediateStorageConfig(None, None)
def config_map_objects( config_value: Any, defs: List[ExecutorDefinition], keyed_by: str, def_type: Type, name_of_def_type: str, ) -> Optional[Dict[str, Any]]: """This function executes the config mappings for executors definitions with respect to ConfigurableDefinition. It calls the ensure_single_item macro on the incoming config and then applies config mapping to the result and the first executor_def with the same name on the mode_def.""" config = config_value.get(keyed_by) check.opt_dict_param(config, "config", key_type=str) if not config: return None obj_name, obj_config = ensure_single_item(config) obj_def = next( (defi for defi in defs if defi.name == obj_name), None ) # obj_defs are stored in a list and we want to find the def matching name check.inst( obj_def, def_type, ("Could not find a {def_type} definition on the selected mode that matches the " '{def_type} "{obj_name}" given in run config').format( def_type=def_type, obj_name=obj_name), ) obj_def = cast(ConfigurableDefinition, obj_def) obj_config_evr = obj_def.apply_config_mapping(obj_config) if not obj_config_evr.success: raise DagsterInvalidConfigError( 'Invalid configuration provided for {} "{}"'.format( name_of_def_type, obj_name), obj_config_evr.errors, obj_config, ) return {obj_name: obj_config_evr.value}
def _selector(context, config_value, runtime_value): selector_key, selector_value = ensure_single_item(config_value) return func(context, selector_key, selector_value, runtime_value)
def test_ensure_single_item(): assert ensure_single_item({"foo": "bar"}) == ("foo", "bar") with pytest.raises(ParameterCheckError, match="Expected dict with single item"): ensure_single_item({"foo": "bar", "baz": "quux"})
def test_ensure_single_item(): assert ensure_single_item({'foo': 'bar'}) == ('foo', 'bar') with pytest.raises(ParameterCheckError, match='Expected dict with single item'): ensure_single_item({'foo': 'bar', 'baz': 'quux'})
def _is_selector_field_optional(config_type: Selector) -> bool: if len(config_type.fields) > 1: return False else: _name, field = ensure_single_item(config_type.fields) return not field.is_required