def get_validated_config(config_yaml=None):
config_type = celery_executor.config_schema.config_type
config_value = get_config_value_from_yaml(config_yaml)
config = validate_config(config_type, config_value)
if not config.success:
raise DagsterInvalidConfigError(
"Errors while loading Celery executor config at {}.".format(
config_yaml),
config.errors,
config_value,
)
return post_process_config(config_type, config_value).value
def get_config_dir(config_yaml=None):
instance = DagsterInstance.get()
config_type = celery_executor.config_field.config_type
config_value = get_config_value_from_yaml(config_yaml)
config_module_name = 'dagster_celery_config'
config_dir = os.path.join(
instance.root_directory, 'dagster_celery', 'config', str(uuid.uuid4())
)
mkdir_p(config_dir)
config_path = os.path.join(
config_dir, '{config_module_name}.py'.format(config_module_name=config_module_name)
)
config = validate_config(config_type, config_value)
if not config.success:
raise DagsterInvalidConfigError(
'Errors while loading Celery executor config at {}.'.format(config_yaml),
config.errors,
config_value,
)
validated_config = post_process_config(config_type, config_value).value
with open(config_path, 'w') as fd:
if 'broker' in validated_config and validated_config['broker']:
fd.write(
'broker_url = \'{broker_url}\'\n'.format(broker_url=str(validated_config['broker']))
)
if 'backend' in validated_config and validated_config['backend']:
fd.write(
'result_backend = \'{result_backend}\'\n'.format(
result_backend=str(validated_config['backend'])
)
)
if 'config_source' in validated_config and validated_config['config_source']:
for key, value in validated_config['config_source'].items():
fd.write('{key} = {value}\n'.format(key=key, value=repr(value)))
# n.b. right now we don't attempt to clean up this cache, but it might make sense to delete
# any files older than some time if there are more than some number of files present, etc.
return config_dir
def get_config_dir(config_yaml=None):
instance = DagsterInstance.get()
config_type = celery_executor.config_schema.config_type
config_value = get_config_value_from_yaml(config_yaml)
config_module_name = "dagster_celery_config"
config_dir = os.path.join(
instance.root_directory, "dagster_celery", "config", str(uuid.uuid4())
)
mkdir_p(config_dir)
config_path = os.path.join(
config_dir, "{config_module_name}.py".format(config_module_name=config_module_name)
)
config = validate_config(config_type, config_value)
if not config.success:
raise DagsterInvalidConfigError(
"Errors while loading Celery executor config at {}.".format(config_yaml),
config.errors,
config_value,
)
validated_config = post_process_config(config_type, config_value).value
with open(config_path, "w") as fd:
if "broker" in validated_config and validated_config["broker"]:
fd.write(
"broker_url = '{broker_url}'\n".format(broker_url=str(validated_config["broker"]))
)
if "backend" in validated_config and validated_config["backend"]:
fd.write(
"result_backend = '{result_backend}'\n".format(
result_backend=str(validated_config["backend"])
)
)
if "config_source" in validated_config and validated_config["config_source"]:
for key, value in validated_config["config_source"].items():
fd.write("{key} = {value}\n".format(key=key, value=repr(value)))
# n.b. right now we don't attempt to clean up this cache, but it might make sense to delete
# any files older than some time if there are more than some number of files present, etc.
return config_dir
def test_post_process_config():
scalar_config_type = resolve_to_config_type(String)
assert post_process_config(scalar_config_type, 'foo').value == 'foo'
assert post_process_config(scalar_config_type, 3).value == 3
assert post_process_config(scalar_config_type, {}).value == {}
assert post_process_config(scalar_config_type, None).value is None
enum_config_type = resolve_to_config_type(
Enum('an_enum', [EnumValue('foo'),
EnumValue('bar', python_value=3)]))
assert post_process_config(enum_config_type, 'foo').value == 'foo'
assert post_process_config(enum_config_type, 'bar').value == 3
assert ('config_value should be pre-validated'
in post_process_config(enum_config_type, 'baz').errors[0].message)
assert ('config_value should be pre-validated'
in post_process_config(enum_config_type, None).errors[0].message)
list_config_type = resolve_to_config_type([str])
assert post_process_config(list_config_type, ['foo']).value == ['foo']
assert post_process_config(list_config_type, None).value == []
with pytest.raises(CheckError, match='Null array member not caught'):
assert post_process_config(list_config_type, [None]).value == [None]
nullable_list_config_type = resolve_to_config_type([Noneable(str)])
assert post_process_config(nullable_list_config_type,
['foo']).value == ['foo']
assert post_process_config(nullable_list_config_type,
[None]).value == [None]
assert post_process_config(nullable_list_config_type, None).value == []
composite_config_type = resolve_to_config_type({
'foo':
String,
'bar': {
'baz': [str]
},
'quux':
Field(str, is_required=False, default_value='zip'),
'quiggle':
Field(str, is_required=False),
})
with pytest.raises(CheckError,
match='Missing non-optional composite member'):
post_process_config(composite_config_type, {})
with pytest.raises(CheckError,
match='Missing non-optional composite member'):
post_process_config(composite_config_type, {
'bar': {
'baz': ['giraffe']
},
'quux': 'nimble'
})
with pytest.raises(CheckError,
match='Missing non-optional composite member'):
post_process_config(composite_config_type, {
'foo': 'zowie',
'quux': 'nimble'
})
assert post_process_config(composite_config_type, {
'foo': 'zowie',
'bar': {
'baz': ['giraffe']
},
'quux': 'nimble'
}).value == {
'foo': 'zowie',
'bar': {
'baz': ['giraffe']
},
'quux': 'nimble'
}
assert post_process_config(composite_config_type, {
'foo': 'zowie',
'bar': {
'baz': ['giraffe']
}
}).value == {
'foo': 'zowie',
'bar': {
'baz': ['giraffe']
},
'quux': 'zip'
}
assert post_process_config(composite_config_type, {
'foo': 'zowie',
'bar': {
'baz': ['giraffe']
},
'quiggle': 'squiggle'
}).value == {
'foo': 'zowie',
'bar': {
'baz': ['giraffe']
},
'quux': 'zip',
'quiggle': 'squiggle'
}
nested_composite_config_type = resolve_to_config_type({
'fruts': {
'apple': Field(String),
'banana': Field(String, is_required=False),
'potato': Field(String, is_required=False, default_value='pie'),
}
})
with pytest.raises(CheckError,
match='Missing non-optional composite member'):
post_process_config(nested_composite_config_type, {'fruts': None})
with pytest.raises(CheckError,
match='Missing non-optional composite member'):
post_process_config(nested_composite_config_type,
{'fruts': {
'banana': 'good',
'potato': 'bad'
}})
assert post_process_config(nested_composite_config_type, {
'fruts': {
'apple': 'strawberry'
}
}).value == {
'fruts': {
'apple': 'strawberry',
'potato': 'pie'
}
}
assert post_process_config(nested_composite_config_type, {
'fruts': {
'apple': 'a',
'banana': 'b',
'potato': 'c'
}
}).value == {
'fruts': {
'apple': 'a',
'banana': 'b',
'potato': 'c'
}
}
any_config_type = resolve_to_config_type(Any)
assert post_process_config(any_config_type, {
'foo': 'bar'
}).value == {
'foo': 'bar'
}
assert post_process_config(
ConfigType('gargle', given_name='bargle', kind=ConfigTypeKind.ANY), 3)
selector_config_type = resolve_to_config_type(
Selector({
'one':
Field(String),
'another': {
'foo': Field(String, default_value='bar', is_required=False)
},
'yet_another':
Field(String, default_value='quux', is_required=False),
}))
with pytest.raises(CheckError):
post_process_config(selector_config_type, 'one')
with pytest.raises(ParameterCheckError):
post_process_config(selector_config_type, None)
with pytest.raises(ParameterCheckError,
match='Expected dict with single item'):
post_process_config(selector_config_type, {})
with pytest.raises(CheckError):
post_process_config(selector_config_type, {
'one': 'foo',
'another': 'bar'
})
assert post_process_config(selector_config_type, {
'one': 'foo'
}).value == {
'one': 'foo'
}
assert post_process_config(selector_config_type, {
'one': None
}).value == {
'one': None
}
assert post_process_config(selector_config_type, {
'one': {}
}).value == {
'one': {}
}
assert post_process_config(selector_config_type, {
'another': {}
}).value == {
'another': {
'foo': 'bar'
}
}
singleton_selector_config_type = resolve_to_config_type(
Selector(
{'foo': Field(String, default_value='bar', is_required=False)}))
assert post_process_config(singleton_selector_config_type, None).value == {
'foo': 'bar'
}
permissive_dict_config_type = resolve_to_config_type(
Permissive({
'foo': Field(String),
'bar': Field(String, default_value='baz', is_required=False)
}))
with pytest.raises(CheckError,
match='Missing non-optional composite member'):
post_process_config(permissive_dict_config_type, None)
assert post_process_config(permissive_dict_config_type, {
'foo': 'wow',
'mau': 'mau'
}).value == {
'foo': 'wow',
'bar': 'baz',
'mau': 'mau',
}
def test_post_process_config():
scalar_config_type = resolve_to_config_type(String)
assert post_process_config(scalar_config_type, "foo").value == "foo"
assert post_process_config(scalar_config_type, 3).value == 3
assert post_process_config(scalar_config_type, {}).value == {}
assert post_process_config(scalar_config_type, None).value is None
enum_config_type = resolve_to_config_type(
Enum("an_enum", [EnumValue("foo"),
EnumValue("bar", python_value=3)]))
assert post_process_config(enum_config_type, "foo").value == "foo"
assert post_process_config(enum_config_type, "bar").value == 3
with pytest.raises(CheckError):
post_process_config(enum_config_type, "baz")
with pytest.raises(CheckError):
post_process_config(enum_config_type, None)
list_config_type = resolve_to_config_type([str])
assert post_process_config(list_config_type, ["foo"]).value == ["foo"]
assert post_process_config(list_config_type, None).value == []
with pytest.raises(CheckError, match="Null array member not caught"):
assert post_process_config(list_config_type, [None]).value == [None]
nullable_list_config_type = resolve_to_config_type([Noneable(str)])
assert post_process_config(nullable_list_config_type,
["foo"]).value == ["foo"]
assert post_process_config(nullable_list_config_type,
[None]).value == [None]
assert post_process_config(nullable_list_config_type, None).value == []
map_config_type = resolve_to_config_type({str: int})
assert post_process_config(map_config_type, {"foo": 5}).value == {"foo": 5}
assert post_process_config(map_config_type, None).value == {}
with pytest.raises(CheckError, match="Null map member not caught"):
assert post_process_config(map_config_type, {
"foo": None
}).value == {
"foo": None
}
nullable_map_config_type = resolve_to_config_type({str: Noneable(int)})
assert post_process_config(nullable_map_config_type, {
"foo": 5
}).value == {
"foo": 5
}
assert post_process_config(nullable_map_config_type, {
"foo": None
}).value == {
"foo": None
}
assert post_process_config(nullable_map_config_type, None).value == {}
composite_config_type = resolve_to_config_type({
"foo":
String,
"bar": {
"baz": [str]
},
"quux":
Field(str, is_required=False, default_value="zip"),
"quiggle":
Field(str, is_required=False),
"werty":
Field({str: [int]}, is_required=False),
})
with pytest.raises(CheckError, match="Missing required composite member"):
post_process_config(composite_config_type, {})
with pytest.raises(CheckError, match="Missing required composite member"):
post_process_config(composite_config_type, {
"bar": {
"baz": ["giraffe"]
},
"quux": "nimble"
})
with pytest.raises(CheckError, match="Missing required composite member"):
post_process_config(composite_config_type, {
"foo": "zowie",
"quux": "nimble"
})
assert post_process_config(composite_config_type, {
"foo": "zowie",
"bar": {
"baz": ["giraffe"]
},
"quux": "nimble"
}).value == {
"foo": "zowie",
"bar": {
"baz": ["giraffe"]
},
"quux": "nimble"
}
assert post_process_config(composite_config_type, {
"foo": "zowie",
"bar": {
"baz": ["giraffe"]
}
}).value == {
"foo": "zowie",
"bar": {
"baz": ["giraffe"]
},
"quux": "zip"
}
assert post_process_config(composite_config_type, {
"foo": "zowie",
"bar": {
"baz": ["giraffe"]
},
"quiggle": "squiggle"
}).value == {
"foo": "zowie",
"bar": {
"baz": ["giraffe"]
},
"quux": "zip",
"quiggle": "squiggle"
}
assert post_process_config(
composite_config_type,
{
"foo": "zowie",
"bar": {
"baz": ["giraffe"]
},
"quiggle": "squiggle",
"werty": {
"asdf": [1, 2, 3]
},
},
).value == {
"foo": "zowie",
"bar": {
"baz": ["giraffe"]
},
"quux": "zip",
"quiggle": "squiggle",
"werty": {
"asdf": [1, 2, 3]
},
}
nested_composite_config_type = resolve_to_config_type({
"fruts": {
"apple": Field(String),
"banana": Field(String, is_required=False),
"potato": Field(String, is_required=False, default_value="pie"),
}
})
with pytest.raises(CheckError, match="Missing required composite member"):
post_process_config(nested_composite_config_type, {"fruts": None})
with pytest.raises(CheckError, match="Missing required composite member"):
post_process_config(nested_composite_config_type,
{"fruts": {
"banana": "good",
"potato": "bad"
}})
assert post_process_config(nested_composite_config_type, {
"fruts": {
"apple": "strawberry"
}
}).value == {
"fruts": {
"apple": "strawberry",
"potato": "pie"
}
}
assert post_process_config(nested_composite_config_type, {
"fruts": {
"apple": "a",
"banana": "b",
"potato": "c"
}
}).value == {
"fruts": {
"apple": "a",
"banana": "b",
"potato": "c"
}
}
any_config_type = resolve_to_config_type(Any)
assert post_process_config(any_config_type, {
"foo": "bar"
}).value == {
"foo": "bar"
}
assert post_process_config(
ConfigType("gargle", given_name="bargle", kind=ConfigTypeKind.ANY), 3)
selector_config_type = resolve_to_config_type(
Selector({
"one":
Field(String),
"another": {
"foo": Field(String, default_value="bar", is_required=False)
},
"yet_another":
Field(String, default_value="quux", is_required=False),
}))
with pytest.raises(CheckError):
post_process_config(selector_config_type, "one")
with pytest.raises(ParameterCheckError):
post_process_config(selector_config_type, None)
with pytest.raises(ParameterCheckError,
match="Expected dict with single item"):
post_process_config(selector_config_type, {})
with pytest.raises(CheckError):
post_process_config(selector_config_type, {
"one": "foo",
"another": "bar"
})
assert post_process_config(selector_config_type, {
"one": "foo"
}).value == {
"one": "foo"
}
assert post_process_config(selector_config_type, {
"one": None
}).value == {
"one": None
}
assert post_process_config(selector_config_type, {
"one": {}
}).value == {
"one": {}
}
assert post_process_config(selector_config_type, {
"another": {}
}).value == {
"another": {
"foo": "bar"
}
}
singleton_selector_config_type = resolve_to_config_type(
Selector(
{"foo": Field(String, default_value="bar", is_required=False)}))
assert post_process_config(singleton_selector_config_type, None).value == {
"foo": "bar"
}
permissive_dict_config_type = resolve_to_config_type(
Permissive({
"foo": Field(String),
"bar": Field(String, default_value="baz", is_required=False)
}))
with pytest.raises(CheckError, match="Missing required composite member"):
post_process_config(permissive_dict_config_type, None)
assert post_process_config(permissive_dict_config_type, {
"foo": "wow",
"mau": "mau"
}).value == {
"foo": "wow",
"bar": "baz",
"mau": "mau",
}
noneable_permissive_config_type = resolve_to_config_type({
"args":
Field(Noneable(Permissive()), is_required=False, default_value=None)
})
assert post_process_config(noneable_permissive_config_type, {
"args": {
"foo": "wow",
"mau": "mau"
}
}).value["args"] == {
"foo": "wow",
"mau": "mau",
}
assert post_process_config(noneable_permissive_config_type, {
"args": {}
}).value["args"] == {}
assert post_process_config(noneable_permissive_config_type,
None).value["args"] == None
