def define_test_type_pipeline():
return PipelineDefinition(
name='test_type_pipeline',
solid_defs=[
define_solid_for_test_type('int_config', int),
define_solid_for_test_type('list_of_int_config', [int]),
define_solid_for_test_type('nullable_list_of_int_config',
Noneable([int])),
define_solid_for_test_type('list_of_nullable_int_config',
[Noneable(int)]),
define_solid_for_test_type('nullable_list_of_nullable_int_config',
Noneable([Noneable(int)])),
define_solid_for_test_type('simple_dict', {
'int_field': int,
'string_field': str
}),
define_solid_for_test_type(
'dict_with_optional_field',
{
'nullable_int_field': Noneable(int),
'optional_int_field': Field(int, is_required=False),
'string_list_field': [str],
},
),
define_solid_for_test_type('nested_dict',
{'nested': {
'int_field': int
}}),
],
)
def shell_solid_config():
return {
'env': Field(
Noneable(Permissive()),
default_value=os.environ.copy(),
is_required=False,
description='An optional dict of environment variables to pass to the subprocess. '
'Defaults to using os.environ.copy().',
),
'output_logging': Field(
Enum(
name='OutputType',
enum_values=[
EnumValue('STREAM', description='Stream script stdout/stderr.'),
EnumValue(
'BUFFER',
description='Buffer shell script stdout/stderr, then log upon completion.',
),
EnumValue('NONE', description='No logging'),
],
),
is_required=False,
default_value='BUFFER',
),
'cwd': Field(
Noneable(str),
default_value=None,
is_required=False,
description='Working directory in which to execute shell script',
),
}
def test_nullable_dict():
dict_with_int = Shape({'int_field': Int})
assert not eval_config_value_from_dagster_type(dict_with_int, None).success
assert not eval_config_value_from_dagster_type(dict_with_int, {}).success
assert not eval_config_value_from_dagster_type(dict_with_int, {'int_field': None}).success
assert eval_config_value_from_dagster_type(dict_with_int, {'int_field': 1}).success
nullable_dict_with_int = Noneable(Shape({'int_field': Int}))
assert eval_config_value_from_dagster_type(nullable_dict_with_int, None).success
assert not eval_config_value_from_dagster_type(nullable_dict_with_int, {}).success
assert not eval_config_value_from_dagster_type(
nullable_dict_with_int, {'int_field': None}
).success
assert eval_config_value_from_dagster_type(nullable_dict_with_int, {'int_field': 1}).success
dict_with_nullable_int = Shape({'int_field': Field(Noneable(int))})
assert not eval_config_value_from_dagster_type(dict_with_nullable_int, None).success
assert not eval_config_value_from_dagster_type(dict_with_nullable_int, {}).success
assert eval_config_value_from_dagster_type(dict_with_nullable_int, {'int_field': None}).success
assert eval_config_value_from_dagster_type(dict_with_nullable_int, {'int_field': 1}).success
nullable_dict_with_nullable_int = Noneable(Shape({'int_field': Field(Noneable(int))}))
assert eval_config_value_from_dagster_type(nullable_dict_with_nullable_int, None).success
assert not eval_config_value_from_dagster_type(nullable_dict_with_nullable_int, {}).success
assert eval_config_value_from_dagster_type(
nullable_dict_with_nullable_int, {'int_field': None}
).success
assert eval_config_value_from_dagster_type(
nullable_dict_with_nullable_int, {'int_field': 1}
).success
def test_nullable_list():
list_of_ints = [int]
assert not eval_config_value_from_dagster_type(list_of_ints, None).success
assert eval_config_value_from_dagster_type(list_of_ints, []).success
assert not eval_config_value_from_dagster_type(list_of_ints, [None]).success
assert eval_config_value_from_dagster_type(list_of_ints, [1]).success
nullable_list_of_ints = Noneable([int])
assert eval_config_value_from_dagster_type(nullable_list_of_ints, None).success
assert eval_config_value_from_dagster_type(nullable_list_of_ints, []).success
assert not eval_config_value_from_dagster_type(nullable_list_of_ints, [None]).success
assert eval_config_value_from_dagster_type(nullable_list_of_ints, [1]).success
list_of_nullable_ints = [Noneable(int)]
assert not eval_config_value_from_dagster_type(list_of_nullable_ints, None).success
assert eval_config_value_from_dagster_type(list_of_nullable_ints, []).success
assert eval_config_value_from_dagster_type(list_of_nullable_ints, [None]).success
assert eval_config_value_from_dagster_type(list_of_nullable_ints, [1]).success
nullable_list_of_nullable_ints = Noneable([Noneable(int)])
assert eval_config_value_from_dagster_type(nullable_list_of_nullable_ints, None).success
assert eval_config_value_from_dagster_type(nullable_list_of_nullable_ints, []).success
assert eval_config_value_from_dagster_type(nullable_list_of_nullable_ints, [None]).success
assert eval_config_value_from_dagster_type(nullable_list_of_nullable_ints, [1]).success
def test_nullable_list():
list_of_ints = [int]
assert not validate_config(list_of_ints, None).success
assert validate_config(list_of_ints, []).success
assert not validate_config(list_of_ints, [None]).success
assert validate_config(list_of_ints, [1]).success
nullable_list_of_ints = Noneable([int])
assert validate_config(nullable_list_of_ints, None).success
assert validate_config(nullable_list_of_ints, []).success
assert not validate_config(nullable_list_of_ints, [None]).success
assert validate_config(nullable_list_of_ints, [1]).success
list_of_nullable_ints = [Noneable(int)]
assert not validate_config(list_of_nullable_ints, None).success
assert validate_config(list_of_nullable_ints, []).success
assert validate_config(list_of_nullable_ints, [None]).success
assert validate_config(list_of_nullable_ints, [1]).success
nullable_list_of_nullable_ints = Noneable([Noneable(int)])
assert validate_config(nullable_list_of_nullable_ints, None).success
assert validate_config(nullable_list_of_nullable_ints, []).success
assert validate_config(nullable_list_of_nullable_ints, [None]).success
assert validate_config(nullable_list_of_nullable_ints, [1]).success
def shell_op_config():
return {
"env": Field(
Noneable(Permissive()),
is_required=False,
description="An optional dict of environment variables to pass to the subprocess.",
),
"output_logging": Field(
Enum(
name="OutputType",
enum_values=[
EnumValue("STREAM", description="Stream script stdout/stderr."),
EnumValue(
"BUFFER",
description="Buffer shell script stdout/stderr, then log upon completion.",
),
EnumValue("NONE", description="No logging"),
],
),
is_required=False,
default_value="BUFFER",
),
"cwd": Field(
Noneable(str),
default_value=None,
is_required=False,
description="Working directory in which to execute shell script",
),
}
def config_type(cls):
return {
'service_account_name':
str,
'job_image':
str,
'instance_config_map':
str,
'postgres_password_secret':
str,
'dagster_home':
str,
'image_pull_secrets':
Field(Noneable(list), is_required=False),
'image_pull_policy':
Field(str, is_required=False, default_value='Always'),
'job_namespace':
str,
'env_config_maps':
Field(Noneable(list), is_required=False),
'env_secrets':
Field(Noneable(list), is_required=False),
'load_kubeconfig':
Field(bool, is_required=False, default_value=False),
'kubeconfig_file':
Field(Noneable(str), is_required=False, default_value=None),
}
def test_resource_invocation_kitchen_sink_config():
@resource(
config_schema={
"str_field": str,
"int_field": int,
"list_int": [int],
"list_list_int": [[int]],
"dict_field": {"a_string": str},
"list_dict_field": [{"an_int": int}],
"selector_of_things": Selector(
{"select_list_dict_field": [{"an_int": int}], "select_int": int}
),
"optional_list_of_optional_string": Noneable([Noneable(str)]),
}
)
def kitchen_sink(context):
return context.resource_config
resource_config = {
"str_field": "kjf",
"int_field": 2,
"list_int": [3],
"list_list_int": [[1], [2, 3]],
"dict_field": {"a_string": "kdjfkd"},
"list_dict_field": [{"an_int": 2}, {"an_int": 4}],
"selector_of_things": {"select_int": 3},
"optional_list_of_optional_string": ["foo", None],
}
assert kitchen_sink(build_init_resource_context(config=resource_config)) == resource_config
def define_test_type_pipeline():
return PipelineDefinition(
name="test_type_pipeline",
solid_defs=[
define_solid_for_test_type("int_config", int),
define_solid_for_test_type("list_of_int_config", [int]),
define_solid_for_test_type("nullable_list_of_int_config",
Noneable([int])),
define_solid_for_test_type("list_of_nullable_int_config",
[Noneable(int)]),
define_solid_for_test_type("nullable_list_of_nullable_int_config",
Noneable([Noneable(int)])),
define_solid_for_test_type("simple_dict", {
"int_field": int,
"string_field": str
}),
define_solid_for_test_type(
"dict_with_optional_field",
{
"nullable_int_field": Noneable(int),
"optional_int_field": Field(int, is_required=False),
"string_list_field": [str],
},
),
define_solid_for_test_type("nested_dict",
{"nested": {
"int_field": int
}}),
],
)
def test_nullable_dict():
dict_with_int = Shape({'int_field': int})
assert not validate_config(dict_with_int, None).success
assert not validate_config(dict_with_int, {}).success
assert not validate_config(dict_with_int, {'int_field': None}).success
assert validate_config(dict_with_int, {'int_field': 1}).success
nullable_dict_with_int = Noneable(Shape({'int_field': int}))
assert validate_config(nullable_dict_with_int, None).success
assert not validate_config(nullable_dict_with_int, {}).success
assert not validate_config(nullable_dict_with_int, {
'int_field': None
}).success
assert validate_config(nullable_dict_with_int, {'int_field': 1}).success
dict_with_nullable_int = Shape({'int_field': Field(Noneable(int))})
assert not validate_config(dict_with_nullable_int, None).success
assert not validate_config(dict_with_nullable_int, {}).success
assert validate_config(dict_with_nullable_int, {'int_field': None}).success
assert validate_config(dict_with_nullable_int, {'int_field': 1}).success
nullable_dict_with_nullable_int = Noneable(
Shape({'int_field': Field(Noneable(int))}))
assert validate_config(nullable_dict_with_nullable_int, None).success
assert not validate_config(nullable_dict_with_nullable_int, {}).success
assert validate_config(nullable_dict_with_nullable_int, {
'int_field': None
}).success
assert validate_config(nullable_dict_with_nullable_int, {
'int_field': 1
}).success
def test_nullable_int():
assert not eval_config_value_from_dagster_type(Int, None).success
assert eval_config_value_from_dagster_type(Int, 0).success
assert eval_config_value_from_dagster_type(Int, 1).success
assert eval_config_value_from_dagster_type(Noneable(int), None).success
assert eval_config_value_from_dagster_type(Noneable(int), 0).success
assert eval_config_value_from_dagster_type(Noneable(int), 1).success
def test_nullable_int():
assert not validate_config(int, None).success
assert validate_config(int, 0).success
assert validate_config(int, 1).success
assert validate_config(Noneable(int), None).success
assert validate_config(Noneable(int), 0).success
assert validate_config(Noneable(int), 1).success
def test_nullable_int():
assert not _validate(int, None).success
assert _validate(int, 0).success
assert _validate(int, 1).success
assert _validate(Noneable(int), None).success
assert _validate(Noneable(int), 0).success
assert _validate(Noneable(int), 1).success
def config_type_pipeline_run(cls):
"""Configuration intended to be set at pipeline execution time.
"""
return {
"job_image":
Field(
Noneable(StringSource),
is_required=False,
description=
"Docker image to use for launched task Jobs. If the repository is not "
"loaded from a GRPC server, then this field is required. If the repository is "
"loaded from a GRPC server, then leave this field empty."
'(Ex: "mycompany.com/dagster-k8s-image:latest").',
),
"image_pull_policy":
Field(
StringSource,
is_required=False,
default_value="IfNotPresent",
description=
"Image pull policy to set on the launched task Job Pods. Defaults to "
'"IfNotPresent".',
),
"image_pull_secrets":
Field(
Noneable(Array(Shape({"name": StringSource}))),
is_required=False,
description=
"(Advanced) Specifies that Kubernetes should get the credentials from "
"the Secrets named in this list.",
),
"service_account_name":
Field(
Noneable(StringSource),
is_required=False,
description=
"(Advanced) Override the name of the Kubernetes service account under "
"which to run the Job.",
),
"env_config_maps":
Field(
Noneable(Array(StringSource)),
is_required=False,
description=
"A list of custom ConfigMapEnvSource names from which to draw "
"environment variables (using ``envFrom``) for the Job. Default: ``[]``. See:"
"https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#define-an-environment-variable-for-a-container",
),
"env_secrets":
Field(
Noneable(Array(StringSource)),
is_required=False,
description=
"A list of custom Secret names from which to draw environment "
"variables (using ``envFrom``) for the Job. Default: ``[]``. See:"
"https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#configure-all-key-value-pairs-in-a-secret-as-container-environment-variables",
),
}
def test_kitchen_sink():
@solid(
config_schema={
"str_field": str,
"int_field": int,
"list_int": [int],
"list_list_int": [[int]],
"dict_field": {"a_string": str},
"list_dict_field": [{"an_int": int}],
"selector_of_things": Selector(
{"select_list_dict_field": [{"an_int": int}], "select_int": int}
),
# this is a good argument to use () instead of [] for type parameterization in
# the config system
"optional_list_of_optional_string": Noneable([Noneable(str)]),
}
)
def kitchen_sink(context):
return context.solid_config
solid_config_one = {
"str_field": "kjf",
"int_field": 2,
"list_int": [3],
"list_list_int": [[1], [2, 3]],
"dict_field": {"a_string": "kdjfkd"},
"list_dict_field": [{"an_int": 2}, {"an_int": 4}],
"selector_of_things": {"select_int": 3},
"optional_list_of_optional_string": ["foo", None],
}
assert (
execute_solid(
kitchen_sink, run_config={"solids": {"kitchen_sink": {"config": solid_config_one}}},
).output_value()
== solid_config_one
)
solid_config_two = {
"str_field": "kjf",
"int_field": 2,
"list_int": [3],
"list_list_int": [[1], [2, 3]],
"dict_field": {"a_string": "kdjfkd"},
"list_dict_field": [{"an_int": 2}, {"an_int": 4}],
"selector_of_things": {"select_list_dict_field": [{"an_int": 5}]},
"optional_list_of_optional_string": None,
}
assert (
execute_solid(
kitchen_sink, run_config={"solids": {"kitchen_sink": {"config": solid_config_two}}},
).output_value()
== solid_config_two
)
def config_type_pipeline_run(cls):
'''Configuration intended to be set at pipeline execution time.
'''
return {
'job_image':
Field(
StringSource,
is_required=True,
description='Docker image to use for launched task Jobs '
'(e.g. "mycompany.com/dagster-k8s-image:latest").',
),
'image_pull_policy':
Field(
StringSource,
is_required=False,
default_value='IfNotPresent',
description=
'Image pull policy to set on the launched task Job Pods. Defaults to '
'"IfNotPresent".',
),
'image_pull_secrets':
Field(
Noneable(Array(Shape({'name': StringSource}))),
is_required=False,
description=
'(Advanced) Specifies that Kubernetes should get the credentials from '
'the Secrets named in this list.',
),
'service_account_name':
Field(
Noneable(StringSource),
is_required=False,
description=
'(Advanced) Override the name of the Kubernetes service account under '
'which to run the Job.',
),
'env_config_maps':
Field(
Noneable(Array(StringSource)),
is_required=False,
description=
'A list of custom ConfigMapEnvSource names from which to draw '
'environment variables (using ``envFrom``) for the Job. Default: ``[]``. See:'
'https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#define-an-environment-variable-for-a-container',
),
'env_secrets':
Field(
Noneable(Array(StringSource)),
is_required=False,
description=
'A list of custom Secret names from which to draw environment '
'variables (using ``envFrom``) for the Job. Default: ``[]``. See:'
'https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#configure-all-key-value-pairs-in-a-secret-as-container-environment-variables',
),
}
def test_kitchen_sink():
@solid(
config_schema={
'str_field': str,
'int_field': int,
'list_int': [int],
'list_list_int': [[int]],
'dict_field': {'a_string': str},
'list_dict_field': [{'an_int': int}],
'selector_of_things': Selector(
{'select_list_dict_field': [{'an_int': int}], 'select_int': int}
),
# this is a good argument to use () instead of [] for type parameterization in
# the config system
'optional_list_of_optional_string': Noneable([Noneable(str)]),
}
)
def kitchen_sink(context):
return context.solid_config
solid_config_one = {
'str_field': 'kjf',
'int_field': 2,
'list_int': [3],
'list_list_int': [[1], [2, 3]],
'dict_field': {'a_string': 'kdjfkd'},
'list_dict_field': [{'an_int': 2}, {'an_int': 4}],
'selector_of_things': {'select_int': 3},
'optional_list_of_optional_string': ['foo', None],
}
assert (
execute_solid(
kitchen_sink, run_config={'solids': {'kitchen_sink': {'config': solid_config_one}}},
).output_value()
== solid_config_one
)
solid_config_two = {
'str_field': 'kjf',
'int_field': 2,
'list_int': [3],
'list_list_int': [[1], [2, 3]],
'dict_field': {'a_string': 'kdjfkd'},
'list_dict_field': [{'an_int': 2}, {'an_int': 4}],
'selector_of_things': {'select_list_dict_field': [{'an_int': 5}]},
'optional_list_of_optional_string': None,
}
assert (
execute_solid(
kitchen_sink, run_config={'solids': {'kitchen_sink': {'config': solid_config_two}}},
).output_value()
== solid_config_two
)
def test_map_failing():
with pytest.raises(ParameterCheckError):
_validate(
Field(Map(key_type="asdf", inner_type=str)),
{
"field_one": "value_one",
"field_two": 2,
},
)
with pytest.raises(ParameterCheckError) as e:
_validate(
Field(Map(Noneable(str), str)),
{
"field_one": "value_one",
"field_two": 2,
},
)
assert "must be a scalar" in str(e)
with pytest.raises(DagsterInvalidDefinitionError) as e:
_validate(
Field({55: str}),
{
"field_one": "value_one",
"field_two": 2,
},
)
assert "Invalid key" in str(e)
with pytest.raises(DagsterInvalidDefinitionError) as e:
_validate(
Field({Noneable(str): str}),
{
"field_one": "value_one",
"field_two": 2,
},
)
assert "Non-scalar key" in str(e)
with pytest.raises(AssertionError):
_validate(
Field(Map(key_type=str, inner_type=str)),
{
"field_one": "value_one",
"field_two": 2,
},
)
def test_kitchen_sink():
kitchen_sink = resolve_to_config_type([{
"opt_list_of_int":
Field(int, is_required=False),
"nested_dict": {
"list_list": [[int]],
"nested_selector":
Field(Selector({
"some_field": int,
"more_list": Noneable([bool])
})),
},
"map": {
str: {
"map_a": int,
"map_b": [str]
},
},
}])
kitchen_sink_snap = snap_from_dagster_type(kitchen_sink)
rehydrated_snap = deserialize_json_to_dagster_namedtuple(
serialize_dagster_namedtuple(kitchen_sink_snap))
assert kitchen_sink_snap == rehydrated_snap
def test_noneable_string_source_array():
assert process_config(Noneable(Array(StringSource)), []).success
assert process_config(Noneable(Array(StringSource)), None).success
assert (
'You have attempted to fetch the environment variable "DAGSTER_TEST_ENV_VAR" '
"which is not set. In order for this execution to succeed it must be set in "
"this environment.") in process_config(Noneable(
Array(StringSource)), ["test", {
"env": "DAGSTER_TEST_ENV_VAR"
}]).errors[0].message
with environ({"DAGSTER_TEST_ENV_VAR": "baz"}):
assert process_config(Noneable(Array(StringSource)),
["test", {
"env": "DAGSTER_TEST_ENV_VAR"
}]).success
def test_kitchen_sink_break_out():
@solid(config_schema=[{
"opt_list_of_int": Field([int], is_required=False),
"nested_dict": {
"list_list": [[int]],
"nested_selector":
Selector({
"some_field": int,
"noneable_list": Noneable([bool])
}),
},
"map": {
str: {
"map_a": int,
"map_b": [str]
},
},
}])
def solid_with_kitchen_sink_config(_):
pass
@pipeline
def single_solid_pipeline():
solid_with_kitchen_sink_config()
config_snaps = build_config_schema_snapshot(
single_solid_pipeline).all_config_snaps_by_key
solid_config_key = solid_with_kitchen_sink_config.config_schema.config_type.key
assert solid_config_key in config_snaps
solid_config_snap = config_snaps[solid_config_key]
assert solid_config_snap.kind == ConfigTypeKind.ARRAY
dict_within_list = config_snaps[solid_config_snap.inner_type_key]
assert len(dict_within_list.fields) == 3
opt_field = dict_within_list.get_field("opt_list_of_int")
assert opt_field.is_required is False
assert config_snaps[opt_field.type_key].kind == ConfigTypeKind.ARRAY
nested_dict = config_snaps[dict_within_list.get_field(
"nested_dict").type_key]
assert len(nested_dict.fields) == 2
nested_selector = config_snaps[nested_dict.get_field(
"nested_selector").type_key]
noneable_list_bool = config_snaps[nested_selector.get_field(
"noneable_list").type_key]
assert noneable_list_bool.kind == ConfigTypeKind.NONEABLE
list_bool = config_snaps[noneable_list_bool.inner_type_key]
assert list_bool.kind == ConfigTypeKind.ARRAY
map = config_snaps[dict_within_list.get_field("map").type_key]
assert map.kind == ConfigTypeKind.MAP
map_dict = config_snaps[map.inner_type_key]
assert len(map_dict.fields) == 2
map_a = config_snaps[map_dict.get_field("map_a").type_key]
assert map_a.kind == ConfigTypeKind.SCALAR
def more_complicated_nested_config():
@solid(
name="a_solid_with_multilayered_config",
input_defs=[],
output_defs=[],
config_schema={
"field_any":
Any,
"field_one":
String,
"field_two":
Field(String, is_required=False),
"field_three":
Field(String, is_required=False, default_value="some_value"),
"nested_field": {
"field_four_str":
String,
"field_five_int":
Int,
"field_six_nullable_int_list":
Field([Noneable(int)], is_required=False),
},
},
)
def a_solid_with_multilayered_config(_):
return None
a_solid_with_multilayered_config()
def test_kitchen_sink_break_out():
nested_dict_cls = resolve_to_config_type({
'list_list': [[int]],
'nested_selector':
Selector({
'some_field': int,
'list': Noneable([bool])
}),
})
dict_within_list_cls = resolve_to_config_type({
'opt_list_of_int':
Field([int], is_optional=True),
'nested_dict':
Field(nested_dict_cls)
})
kitchen_sink = Array(dict_within_list_cls)
dict_within_list_key = dict_within_list_cls.key
kitchen_sink_meta = meta_from_dagster_type(kitchen_sink)
assert len(kitchen_sink_meta.type_param_refs) == 1
assert kitchen_sink_meta.type_param_refs[0].key == dict_within_list_key
assert len(kitchen_sink_meta.inner_type_refs) == 1
assert kitchen_sink_meta.inner_type_refs[0].key == dict_within_list_key
dict_within_list_meta = meta_from_dagster_type(dict_within_list_cls)
assert dict_within_list_meta.type_param_refs is None
# List[int], Int, Shape.XXX
assert len(dict_within_list_meta.inner_type_refs) == 3
assert sorted([
type_ref.key for type_ref in dict_within_list_meta.inner_type_refs
]) == sorted([nested_dict_cls.key, 'Int', 'Array.Int'])
def more_complicated_nested_config():
@solid(
name='a_solid_with_multilayered_config',
input_defs=[],
output_defs=[],
config={
'field_any':
Any,
'field_one':
String,
'field_two':
Field(String, is_optional=True),
'field_three':
Field(String, is_optional=True, default_value='some_value'),
'nested_field': {
'field_four_str':
String,
'field_five_int':
Int,
'field_six_nullable_int_list':
Field([Noneable(int)], is_optional=True),
},
},
)
def a_solid_with_multilayered_config(_):
return None
return a_solid_with_multilayered_config()
def celery_k8s_config():
from dagster_k8s import DagsterK8sJobConfig
# DagsterK8sJobConfig provides config schema for specifying Dagster K8s Jobs
job_config = DagsterK8sJobConfig.config_type_pipeline_run()
additional_config = {
'load_incluster_config': Field(
bool,
is_required=False,
default_value=True,
description='''Set this value if you are running the launcher within a k8s cluster. If
``True``, we assume the launcher is running within the target cluster and load config
using ``kubernetes.config.load_incluster_config``. Otherwise, we will use the k8s config
specified in ``kubeconfig_file`` (using ``kubernetes.config.load_kube_config``) or fall
back to the default kubeconfig. Default: ``True``.''',
),
'kubeconfig_file': Field(
Noneable(str),
is_required=False,
description='Path to a kubeconfig file to use, if not using default kubeconfig.',
),
'job_namespace': Field(
StringSource,
is_required=False,
default_value='default',
description='The namespace into which to launch new jobs. Note that any '
'other Kubernetes resources the Job requires (such as the service account) must be '
'present in this namespace. Default: ``"default"``',
),
}
cfg = merge_dicts(CELERY_CONFIG, job_config)
cfg = merge_dicts(cfg, additional_config)
return cfg
def test_noneable_string_source_array():
assert process_config(Noneable(Array(StringSource)), []).success
assert process_config(Noneable(Array(StringSource)), None).success
assert ((
'You have attempted to fetch the environment variable "DAGSTER_TEST_ENV_VAR" '
'which is not set. In order for this execution to succeed it must be set in '
'this environment.')
in process_config(Noneable(Array(StringSource)),
['test', {
'env': 'DAGSTER_TEST_ENV_VAR'
}]).errors[0].message)
with environ({'DAGSTER_TEST_ENV_VAR': 'baz'}):
assert process_config(Noneable(Array(StringSource)),
['test', {
'env': 'DAGSTER_TEST_ENV_VAR'
}]).success
def test_invalid_list_element():
with pytest.raises(
DagsterInvalidDefinitionError,
match=re.escape(
"Invalid type: dagster_type must be an instance of DagsterType or a Python type: "
),
):
_ = List[Noneable(int)]
def test_non_scalar_key_map():
with pytest.raises(
DagsterInvalidConfigDefinitionError,
match=re.escape("Map dict must have a scalar type as its only key."),
):
@solid(config_schema={Noneable(int): str})
def _solid(_):
pass
def config_type(cls):
job_cfg = DagsterK8sJobConfig.config_type()
scheduler_extra_cfg = {
'scheduler_namespace': Field(StringSource, is_required=True),
'load_incluster_config': Field(bool, is_required=False, default_value=True),
'kubeconfig_file': Field(Noneable(StringSource), is_required=False, default_value=None),
}
return merge_dicts(job_cfg, scheduler_extra_cfg)
def test_list_nullable_int():
lni = resolve_to_config_type(Array(Noneable(int)))
assert validate_config(lni, [1]).success
assert validate_config(lni, [1, 2]).success
assert validate_config(lni, []).success
assert validate_config(lni, [None]).success
assert validate_config(lni, [1, None]).success
assert not validate_config(lni, None).success
assert not validate_config(lni, [1, 'absdf']).success
