def test_kitchen_sink_break_out():
nested_dict_cls = resolve_to_config_type({
'list_list': [[int]],
'nested_selector':
Selector({
'some_field': int,
'list': Noneable([bool])
}),
})
dict_within_list_cls = resolve_to_config_type({
'opt_list_of_int':
Field([int], is_optional=True),
'nested_dict':
Field(nested_dict_cls)
})
kitchen_sink = Array(dict_within_list_cls)
dict_within_list_key = dict_within_list_cls.key
kitchen_sink_meta = meta_from_dagster_type(kitchen_sink)
assert len(kitchen_sink_meta.type_param_refs) == 1
assert kitchen_sink_meta.type_param_refs[0].key == dict_within_list_key
assert len(kitchen_sink_meta.inner_type_refs) == 1
assert kitchen_sink_meta.inner_type_refs[0].key == dict_within_list_key
dict_within_list_meta = meta_from_dagster_type(dict_within_list_cls)
assert dict_within_list_meta.type_param_refs is None
# List[int], Int, Shape.XXX
assert len(dict_within_list_meta.inner_type_refs) == 3
assert sorted([
type_ref.key for type_ref in dict_within_list_meta.inner_type_refs
]) == sorted([nested_dict_cls.key, 'Int', 'Array.Int'])
def test_scalar_union():
non_scalar_type = {'str_field': String}
scalar_union_type = ScalarUnion(
scalar_type=resolve_to_config_type(int),
non_scalar_type=resolve_to_config_type(non_scalar_type),
)
assert_inner_types(scalar_union_type, String, Int, non_scalar_type)
def test_scalar_or_list():
int_or_list = ScalarUnion(scalar_type=resolve_to_config_type(int),
non_scalar_type=resolve_to_config_type([str]))
assert validate_config(int_or_list, 2).success
assert not validate_config(int_or_list, '2').success
assert not validate_config(int_or_list, False).success
assert validate_config(int_or_list, []).success
assert validate_config(int_or_list, ['ab']).success
assert not validate_config(int_or_list, [2]).success
assert not validate_config(int_or_list, {}).success
def output_materialization_config(config_cls, required_resource_keys=None):
'''Create an output materialization hydration config that configurably materializes a runtime
value.
The decorated function should take the execution context, the parsed config value, and the
runtime value and the parsed config data, should materialize the runtime value, and should
return an appropriate :py:class:`Materialization`.
Args:
config_cls (Any): The type of the config data expected by the decorated function. Users
should provide one of the :ref:`built-in types <builtin>`, or a composite constructed
using :py:func:`Selector` or :py:func:`Permissive`.
Examples:
.. code-block:: python
# Takes a list of dicts such as might be read in using csv.DictReader, as well as a config
value, and writes
@output_materialization_config(Path)
def df_output_schema(_context, path, value):
with open(path, 'w') as fd:
writer = csv.DictWriter(fd, fieldnames=value[0].keys())
writer.writeheader()
writer.writerows(rowdicts=value)
return Materialization.file(path)
'''
from dagster.config.field import resolve_to_config_type
config_type = resolve_to_config_type(config_cls)
return lambda func: _create_output_schema(config_type, func,
required_resource_keys)
def _get_validated_celery_k8s_executor_config(run_config):
check.dict_param(run_config, "run_config")
executor_config = run_config.get("execution", {})
execution_config_schema = resolve_to_config_type(celery_k8s_config())
# In run config on jobs, we don't have an executor key
if not CELERY_K8S_CONFIG_KEY in executor_config:
execution_run_config = executor_config.get("config", {})
else:
execution_run_config = (run_config["execution"][CELERY_K8S_CONFIG_KEY]
or {}).get("config", {})
res = process_config(execution_config_schema, execution_run_config)
check.invariant(
res.success,
"Incorrect execution schema provided. Note: You may also be seeing this error "
"because you are using the configured API. "
"Using configured with the {config_key} executor is not supported at this time, "
"and all executor config must be directly in the run config without using configured."
.format(config_key=CELERY_K8S_CONFIG_KEY, ),
)
return res.value
def _get_validated_celery_k8s_executor_config(environment_dict):
check.dict_param(environment_dict, 'environment_dict')
try:
from dagster_celery.executor_k8s import CELERY_K8S_CONFIG_KEY, celery_k8s_config
except ImportError:
raise DagsterK8sError(
'To use the CeleryK8sRunLauncher, dagster_celery must be'
' installed in your Python environment.'
)
check.invariant(
CELERY_K8S_CONFIG_KEY in environment_dict.get('execution', {}),
'{} execution must be configured in pipeline execution config to launch runs with '
'CeleryK8sRunLauncher'.format(CELERY_K8S_CONFIG_KEY),
)
execution_config_schema = resolve_to_config_type(celery_k8s_config())
execution_run_config = environment_dict['execution'][CELERY_K8S_CONFIG_KEY].get('config', {})
res = process_config(execution_config_schema, execution_run_config)
check.invariant(
res.success, 'Incorrect {} execution schema provided'.format(CELERY_K8S_CONFIG_KEY)
)
return res.value
def dagster_type_materializer(config_schema, required_resource_keys=None):
'''Create an output materialization hydration config that configurably materializes a runtime
value.
The decorated function should take the execution context, the parsed config value, and the
runtime value and the parsed config data, should materialize the runtime value, and should
return an appropriate :py:class:`AssetMaterialization`.
Args:
config_schema (Any): The type of the config data expected by the decorated function.
Examples:
.. code-block:: python
# Takes a list of dicts such as might be read in using csv.DictReader, as well as a config
value, and writes
@dagster_type_materializer(Path)
def materialize_df(_context, path, value):
with open(path, 'w') as fd:
writer = csv.DictWriter(fd, fieldnames=value[0].keys())
writer.writeheader()
writer.writerows(rowdicts=value)
return AssetMaterialization.file(path)
'''
from dagster.config.field import resolve_to_config_type
config_type = resolve_to_config_type(config_schema)
return lambda func: _create_output_materializer_for_decorator(
config_type, func, required_resource_keys
)
def test_scalar_or_selector():
int_or_selector = ScalarUnion(
scalar_type=resolve_to_config_type(int),
non_scalar_type=Selector({
'a_string': str,
'an_int': int
}),
)
assert validate_config(int_or_selector, 2).success
assert not validate_config(int_or_selector, '2').success
assert not validate_config(int_or_selector, False).success
assert validate_config(int_or_selector, {'a_string': 'kjdfk'}).success
assert validate_config(int_or_selector, {'an_int': 2}).success
assert not validate_config(int_or_selector, {}).success
assert not validate_config(int_or_selector, {
'a_string': 'kjdfk',
'an_int': 2
}).success
assert not validate_config(int_or_selector, {'wrong_key': 'kjdfd'}).success
assert not validate_config(int_or_selector, {'a_string': 2}).success
assert not validate_config(int_or_selector, {
'a_string': 'kjdfk',
'extra_field': 'kd'
}).success
def output_selector_schema(config_cls, required_resource_keys=None):
'''
Deprecated in favor of dagster_type_materializer.
A decorator for a annotating a function that can take the selected properties
of a ``config_value`` and an instance of a custom type and materialize it.
Args:
config_cls (Selector):
'''
rename_warning('dagster_type_materializer', 'output_selector_schema', '0.10.0')
from dagster.config.field import resolve_to_config_type
config_type = resolve_to_config_type(config_cls)
check.param_invariant(config_type.kind == ConfigTypeKind.SELECTOR, 'config_cls')
def _wrap(func):
def _selector(context, config_value, runtime_value):
selector_key, selector_value = ensure_single_item(config_value)
return func(context, selector_key, selector_value, runtime_value)
return _create_output_materializer_for_decorator(
config_type, _selector, required_resource_keys
)
return _wrap
def test_kitchen_sink():
kitchen_sink = resolve_to_config_type([{
"opt_list_of_int":
Field(int, is_required=False),
"nested_dict": {
"list_list": [[int]],
"nested_selector":
Field(Selector({
"some_field": int,
"more_list": Noneable([bool])
})),
},
"map": {
str: {
"map_a": int,
"map_b": [str]
},
},
}])
kitchen_sink_snap = snap_from_dagster_type(kitchen_sink)
rehydrated_snap = deserialize_json_to_dagster_namedtuple(
serialize_dagster_namedtuple(kitchen_sink_snap))
assert kitchen_sink_snap == rehydrated_snap
def rehydrate(self):
from dagster.config.field import resolve_to_config_type
from dagster.config.validate import process_config
from dagster.core.errors import DagsterInvalidConfigError
try:
module = importlib.import_module(self.module_name)
except ModuleNotFoundError:
check.failed(
f"Couldn't import module {self.module_name} when attempting to load the "
f"configurable class {self.module_name}.{self.class_name}")
try:
klass = getattr(module, self.class_name)
except AttributeError:
check.failed(
f"Couldn't find class {self.class_name} in module when attempting to load the "
f"configurable class {self.module_name}.{self.class_name}")
if not issubclass(klass, ConfigurableClass):
raise check.CheckError(
klass,
f"class {self.class_name} in module {self.module_name}",
ConfigurableClass,
)
config_dict = self.config_dict
result = process_config(resolve_to_config_type(klass.config_type()),
config_dict)
if not result.success:
raise DagsterInvalidConfigError(
f"Errors whilst loading configuration for {klass.config_type()}.",
result.errors,
config_dict,
)
return klass.from_config_value(self, result.value)
def input_selector_schema(config_cls, required_resource_keys=None):
"""
Deprecated in favor of dagster_type_loader.
A decorator for annotating a function that can take the selected properties
from a ``config_value`` in to an instance of a custom type.
Args:
config_cls (Selector)
"""
rename_warning("dagster_type_loader", "input_selector_schema", "0.10.0")
from dagster.config.field import resolve_to_config_type
config_type = resolve_to_config_type(config_cls)
check.param_invariant(config_type.kind == ConfigTypeKind.SELECTOR,
"config_cls")
def _wrap(func):
def _selector(context, config_value):
selector_key, selector_value = ensure_single_item(config_value)
return func(context, selector_key, selector_value)
return _create_type_loader_for_decorator(config_type, _selector,
required_resource_keys)
return _wrap
def test_list_of_scalar_or_dict():
int_or_dict_list = resolve_to_config_type(
[
ScalarUnion(
scalar_type=resolve_to_config_type(int), non_scalar_type=Shape({'a_string': str})
)
]
)
assert validate_config(int_or_dict_list, []).success
assert validate_config(int_or_dict_list, [2]).success
assert validate_config(int_or_dict_list, [{'a_string': 'kjdfd'}]).success
assert validate_config(int_or_dict_list, [2, {'a_string': 'kjdfd'}]).success
assert not validate_config(int_or_dict_list, [2, {'wrong_key': 'kjdfd'}]).success
assert not validate_config(int_or_dict_list, [2, {'a_string': 2343}]).success
assert not validate_config(int_or_dict_list, ['kjdfkd', {'a_string': 'kjdfd'}]).success
def assert_inner_types(parent_type, *dagster_types):
assert set(
list(
map(lambda t: t.key,
resolve_to_config_type(
parent_type).recursive_config_types))) == set(
map(lambda x: x.key,
map(resolve_to_config_type, dagster_types)))
def remap_python_builtin_for_config(ttype):
'''This function remaps a python type to a Dagster type, or passes it through if it cannot be
remapped.
'''
from dagster.config.field import resolve_to_config_type
check.param_invariant(is_supported_config_python_builtin(ttype), 'ttype')
return resolve_to_config_type(SUPPORTED_CONFIG_BUILTINS[ttype])
def test_list_int():
list_int = resolve_to_config_type([Int])
assert validate_config(list_int, [1]).success
assert validate_config(list_int, [1, 2]).success
assert validate_config(list_int, []).success
assert not validate_config(list_int, [None]).success
assert not validate_config(list_int, [1, None]).success
assert not validate_config(list_int, None).success
assert not validate_config(list_int, [1, 'absdf']).success
def test_list_nullable_int():
lni = resolve_to_config_type(Array(Noneable(int)))
assert validate_config(lni, [1]).success
assert validate_config(lni, [1, 2]).success
assert validate_config(lni, []).success
assert validate_config(lni, [None]).success
assert validate_config(lni, [1, None]).success
assert not validate_config(lni, None).success
assert not validate_config(lni, [1, 'absdf']).success
def assert_inner_types(parent_type, *dagster_types):
config_type = resolve_to_config_type(parent_type)
config_schema_snapshot = config_schema_snapshot_from_config_type(
config_type)
all_type_keys = get_recursive_type_keys(snap_from_config_type(config_type),
config_schema_snapshot)
assert set(all_type_keys) == set(
map(lambda x: x.key, map(resolve_to_config_type, dagster_types)))
def test_scalar_or_list():
int_or_list = ScalarUnion(scalar_type=int, non_scalar_schema=resolve_to_config_type([str]))
assert validate_config(int_or_list, 2).success
assert not validate_config(int_or_list, "2").success
assert not validate_config(int_or_list, False).success
assert validate_config(int_or_list, []).success
assert validate_config(int_or_list, ["ab"]).success
assert not validate_config(int_or_list, [2]).success
assert not validate_config(int_or_list, {}).success
def test_scalar_union():
# Requiring resolve calls is bad: https://github.com/dagster-io/dagster/issues/2266
@solid(config=ScalarUnion(resolve_to_config_type(str), resolve_to_config_type({'bar': str})))
def solid_with_config(_):
pass
@pipeline
def single_solid_pipeline():
solid_with_config()
config_snaps = build_config_schema_snapshot(single_solid_pipeline).all_config_snaps_by_key
scalar_union_key = solid_with_config.config_field.config_type.key
assert scalar_union_key in config_snaps
assert config_snaps[config_snaps[scalar_union_key].scalar_type_key].key == 'String'
assert (
config_snaps[config_snaps[scalar_union_key].non_scalar_type_key].kind
== ConfigTypeKind.STRICT_SHAPE
)
def dagster_type_loader(
config_schema,
required_resource_keys=None,
loader_version=None,
external_version_fn=lambda x: None,
):
"""Create an dagster type loader that maps config data to a runtime value.
The decorated function should take the execution context and parsed config value and return the
appropriate runtime value.
Args:
config_schema (ConfigSchema): The schema for the config that's passed to the decorated
function.
loader_version (str): (Experimental) The version of the decorated compute function. Two
loading functions should have the same version if and only if they deterministically
produce the same outputs when provided the same inputs.
external_version_fn (Callable): A function that takes in the same parameters as the loader
function (config_value) and returns a representation of the version of the external
asset (str). Two external assets with identical versions are treated as identical to one
another.
Examples:
.. code-block:: python
@dagster_type_loader(Permissive())
def load_dict(_context, value):
return value
"""
from dagster.config.field import resolve_to_config_type
config_type = resolve_to_config_type(config_schema)
EXPECTED_POSITIONALS = ["context", "*"]
def wrapper(func):
fn_positionals, _ = split_function_parameters(func,
EXPECTED_POSITIONALS)
missing_positional = validate_decorated_fn_positionals(
fn_positionals, EXPECTED_POSITIONALS)
if missing_positional:
raise DagsterInvalidDefinitionError(
"@dagster_type_loader '{solid_name}' decorated function does not have required positional "
"parameter '{missing_param}'. Solid functions should only have keyword arguments "
"that match input names and a first positional parameter named 'context'."
.format(solid_name=func.__name__,
missing_param=missing_positional))
return _create_type_loader_for_decorator(config_type, func,
required_resource_keys,
loader_version,
external_version_fn)
return wrapper
def test_list_of_scalar_or_dict():
int_or_dict_list = resolve_to_config_type(
[ScalarUnion(scalar_type=int, non_scalar_schema=Shape({"a_string": str}))]
)
assert validate_config(int_or_dict_list, []).success
assert validate_config(int_or_dict_list, [2]).success
assert validate_config(int_or_dict_list, [{"a_string": "kjdfd"}]).success
assert validate_config(int_or_dict_list, [2, {"a_string": "kjdfd"}]).success
assert not validate_config(int_or_dict_list, [2, {"wrong_key": "kjdfd"}]).success
assert not validate_config(int_or_dict_list, [2, {"a_string": 2343}]).success
assert not validate_config(int_or_dict_list, ["kjdfkd", {"a_string": "kjdfd"}]).success
def test_list_of_dict():
inner_dict_dagster_type = Shape({'foo': Field(str)})
list_of_dict_meta = meta_from_dagster_type([inner_dict_dagster_type])
assert list_of_dict_meta.key.startswith('Array')
assert list_of_dict_meta.inner_type_refs
assert len(list_of_dict_meta.inner_type_refs) == 1
# Both Shape[...] and str are NonGenericTypeRefMetas in this schema
dict_ref = list_of_dict_meta.type_param_refs[0]
assert isinstance(dict_ref, NonGenericTypeRefMeta)
assert dict_ref.key.startswith('Shape')
assert (len(list_of_dict_meta.type_param_refs) == 1
and list_of_dict_meta.type_param_refs[0].key
== resolve_to_config_type(inner_dict_dagster_type).key)
def test_map_int():
map_str_int = resolve_to_config_type({str: Int})
assert validate_config(map_str_int, {"a": 1}).success
assert validate_config(map_str_int, {"a": 1, "b": 2}).success
assert validate_config(map_str_int, {}).success
assert not validate_config(map_str_int, {"a": None}).success
assert not validate_config(map_str_int, {"a": 1, "b": None}).success
assert not validate_config(map_str_int, None).success
assert not validate_config(map_str_int, {"a": 1, "b": "absdf"}).success
assert not validate_config(map_str_int, {"a": 1, 4: 4}).success
assert not validate_config(map_str_int, {"a": 1, None: 4}).success
map_int_int = resolve_to_config_type({Int: Int})
assert validate_config(map_int_int, {1: 1}).success
assert validate_config(map_int_int, {2: 1, 3: 2}).success
assert validate_config(map_int_int, {}).success
assert not validate_config(map_int_int, {1: None}).success
assert not validate_config(map_int_int, {1: 1, 2: None}).success
assert not validate_config(map_int_int, None).success
assert not validate_config(map_int_int, {1: 1, 2: "absdf"}).success
assert not validate_config(map_int_int, {4: 1, "a": 4}).success
assert not validate_config(map_int_int, {5: 1, None: 4}).success
def test_scalar_or_dict():
int_or_dict = ScalarUnion(
scalar_type=resolve_to_config_type(int), non_scalar_type=Shape({'a_string': str})
)
assert validate_config(int_or_dict, 2).success
assert not validate_config(int_or_dict, '2').success
assert not validate_config(int_or_dict, False).success
assert validate_config(int_or_dict, {'a_string': 'kjdfk'}).success
assert not validate_config(int_or_dict, {}).success
assert not validate_config(int_or_dict, {'wrong_key': 'kjdfd'}).success
assert not validate_config(int_or_dict, {'a_string': 2}).success
assert not validate_config(int_or_dict, {'a_string': 'kjdfk', 'extra_field': 'kd'}).success
def dagster_instance_config(base_dir,
config_filename=DAGSTER_CONFIG_YAML_FILENAME,
overrides=None):
overrides = check.opt_dict_param(overrides, 'overrides')
dagster_config_dict = merge_dicts(
load_yaml_from_globs(os.path.join(base_dir, config_filename)),
overrides)
dagster_config_type = resolve_to_config_type(define_dagster_config_cls())
dagster_config = validate_config(dagster_config_type, dagster_config_dict)
if not dagster_config.success:
raise DagsterInvalidConfigError(
'Errors whilst loading dagster instance config at {}.'.format(
config_filename),
dagster_config.errors,
dagster_config_dict,
)
return dagster_config.value
def rehydrate(self):
from dagster.core.errors import DagsterInvalidConfigError
from dagster.config.field import resolve_to_config_type
from dagster.config.validate import process_config
try:
module = importlib.import_module(self.module_name)
except seven.ModuleNotFoundError:
check.failed(
'Couldn\'t import module {module_name} when attempting to rehydrate the '
'configurable class {configurable_class}'.format(
module_name=self.module_name,
configurable_class=self.module_name + '.' +
self.class_name,
))
try:
klass = getattr(module, self.class_name)
except AttributeError:
check.failed(
'Couldn\'t find class {class_name} in module when attempting to rehydrate the '
'configurable class {configurable_class}'.format(
class_name=self.class_name,
configurable_class=self.module_name + '.' +
self.class_name,
))
if not issubclass(klass, ConfigurableClass):
raise check.CheckError(
klass,
'class {class_name} in module {module_name}'.format(
class_name=self.class_name, module_name=self.module_name),
ConfigurableClass,
)
config_dict = yaml.safe_load(self.config_yaml)
result = process_config(resolve_to_config_type(klass.config_type()),
config_dict)
if not result.success:
raise DagsterInvalidConfigError(
'Errors whilst loading configuration for {}.'.format(
klass.config_type()),
result.errors,
config_dict,
)
return klass.from_config_value(self, result.value)
def _get_validated_celery_k8s_executor_config(run_config):
check.dict_param(run_config, "run_config")
check.invariant(
CELERY_K8S_CONFIG_KEY in run_config.get("execution", {}),
"{} execution must be configured in pipeline execution config to launch runs with "
"CeleryK8sRunLauncher".format(CELERY_K8S_CONFIG_KEY),
)
execution_config_schema = resolve_to_config_type(celery_k8s_config())
execution_run_config = run_config["execution"][CELERY_K8S_CONFIG_KEY].get("config", {})
res = process_config(execution_config_schema, execution_run_config)
check.invariant(
res.success, "Incorrect {} execution schema provided".format(CELERY_K8S_CONFIG_KEY)
)
return res.value
def test_kitchen_sink():
kitchen_sink = resolve_to_config_type([{
'opt_list_of_int':
Field(int, is_optional=True),
'nested_dict': {
'list_list': [[int]],
'nested_selector':
Field(Selector({
'some_field': int,
'more_list': Noneable([bool])
})),
},
}])
kitchen_sink_meta = meta_from_dagster_type(kitchen_sink)
rehydrated_meta = deserialize_json_to_dagster_namedtuple(
serialize_dagster_namedtuple(kitchen_sink_meta))
assert kitchen_sink_meta == rehydrated_meta
def input_selector_schema(config_cls, required_resource_keys=None):
'''
A decorator for annotating a function that can take the selected properties
from a ``config_value`` in to an instance of a custom type.
Args:
config_cls (Selector)
'''
from dagster.config.field import resolve_to_config_type
config_type = resolve_to_config_type(config_cls)
check.param_invariant(config_type.kind == ConfigTypeKind.SELECTOR, 'config_cls')
def _wrap(func):
def _selector(context, config_value):
selector_key, selector_value = ensure_single_item(config_value)
return func(context, selector_key, selector_value)
return _create_input_schema_for_decorator(config_type, _selector, required_resource_keys)
return _wrap
