def get_validated_config(config_yaml=None): config_type = celery_executor.config_schema.config_type config_value = get_config_value_from_yaml(config_yaml) config = validate_config(config_type, config_value) if not config.success: raise DagsterInvalidConfigError( "Errors while loading Celery executor config at {}.".format( config_yaml), config.errors, config_value, ) return post_process_config(config_type, config_value).value
def get_config_dir(config_yaml=None): instance = DagsterInstance.get() config_type = celery_executor.config_field.config_type config_value = get_config_value_from_yaml(config_yaml) config_module_name = 'dagster_celery_config' config_dir = os.path.join( instance.root_directory, 'dagster_celery', 'config', str(uuid.uuid4()) ) mkdir_p(config_dir) config_path = os.path.join( config_dir, '{config_module_name}.py'.format(config_module_name=config_module_name) ) config = validate_config(config_type, config_value) if not config.success: raise DagsterInvalidConfigError( 'Errors while loading Celery executor config at {}.'.format(config_yaml), config.errors, config_value, ) validated_config = post_process_config(config_type, config_value).value with open(config_path, 'w') as fd: if 'broker' in validated_config and validated_config['broker']: fd.write( 'broker_url = \'{broker_url}\'\n'.format(broker_url=str(validated_config['broker'])) ) if 'backend' in validated_config and validated_config['backend']: fd.write( 'result_backend = \'{result_backend}\'\n'.format( result_backend=str(validated_config['backend']) ) ) if 'config_source' in validated_config and validated_config['config_source']: for key, value in validated_config['config_source'].items(): fd.write('{key} = {value}\n'.format(key=key, value=repr(value))) # n.b. right now we don't attempt to clean up this cache, but it might make sense to delete # any files older than some time if there are more than some number of files present, etc. return config_dir
def get_config_dir(config_yaml=None): instance = DagsterInstance.get() config_type = celery_executor.config_schema.config_type config_value = get_config_value_from_yaml(config_yaml) config_module_name = "dagster_celery_config" config_dir = os.path.join( instance.root_directory, "dagster_celery", "config", str(uuid.uuid4()) ) mkdir_p(config_dir) config_path = os.path.join( config_dir, "{config_module_name}.py".format(config_module_name=config_module_name) ) config = validate_config(config_type, config_value) if not config.success: raise DagsterInvalidConfigError( "Errors while loading Celery executor config at {}.".format(config_yaml), config.errors, config_value, ) validated_config = post_process_config(config_type, config_value).value with open(config_path, "w") as fd: if "broker" in validated_config and validated_config["broker"]: fd.write( "broker_url = '{broker_url}'\n".format(broker_url=str(validated_config["broker"])) ) if "backend" in validated_config and validated_config["backend"]: fd.write( "result_backend = '{result_backend}'\n".format( result_backend=str(validated_config["backend"]) ) ) if "config_source" in validated_config and validated_config["config_source"]: for key, value in validated_config["config_source"].items(): fd.write("{key} = {value}\n".format(key=key, value=repr(value))) # n.b. right now we don't attempt to clean up this cache, but it might make sense to delete # any files older than some time if there are more than some number of files present, etc. return config_dir
def test_post_process_config(): scalar_config_type = resolve_to_config_type(String) assert post_process_config(scalar_config_type, 'foo').value == 'foo' assert post_process_config(scalar_config_type, 3).value == 3 assert post_process_config(scalar_config_type, {}).value == {} assert post_process_config(scalar_config_type, None).value is None enum_config_type = resolve_to_config_type( Enum('an_enum', [EnumValue('foo'), EnumValue('bar', python_value=3)])) assert post_process_config(enum_config_type, 'foo').value == 'foo' assert post_process_config(enum_config_type, 'bar').value == 3 assert ('config_value should be pre-validated' in post_process_config(enum_config_type, 'baz').errors[0].message) assert ('config_value should be pre-validated' in post_process_config(enum_config_type, None).errors[0].message) list_config_type = resolve_to_config_type([str]) assert post_process_config(list_config_type, ['foo']).value == ['foo'] assert post_process_config(list_config_type, None).value == [] with pytest.raises(CheckError, match='Null array member not caught'): assert post_process_config(list_config_type, [None]).value == [None] nullable_list_config_type = resolve_to_config_type([Noneable(str)]) assert post_process_config(nullable_list_config_type, ['foo']).value == ['foo'] assert post_process_config(nullable_list_config_type, [None]).value == [None] assert post_process_config(nullable_list_config_type, None).value == [] composite_config_type = resolve_to_config_type({ 'foo': String, 'bar': { 'baz': [str] }, 'quux': Field(str, is_required=False, default_value='zip'), 'quiggle': Field(str, is_required=False), }) with pytest.raises(CheckError, match='Missing non-optional composite member'): post_process_config(composite_config_type, {}) with pytest.raises(CheckError, match='Missing non-optional composite member'): post_process_config(composite_config_type, { 'bar': { 'baz': ['giraffe'] }, 'quux': 'nimble' }) with pytest.raises(CheckError, match='Missing non-optional composite member'): post_process_config(composite_config_type, { 'foo': 'zowie', 'quux': 'nimble' }) assert post_process_config(composite_config_type, { 'foo': 'zowie', 'bar': { 'baz': ['giraffe'] }, 'quux': 'nimble' }).value == { 'foo': 'zowie', 'bar': { 'baz': ['giraffe'] }, 'quux': 'nimble' } assert post_process_config(composite_config_type, { 'foo': 'zowie', 'bar': { 'baz': ['giraffe'] } }).value == { 'foo': 'zowie', 'bar': { 'baz': ['giraffe'] }, 'quux': 'zip' } assert post_process_config(composite_config_type, { 'foo': 'zowie', 'bar': { 'baz': ['giraffe'] }, 'quiggle': 'squiggle' }).value == { 'foo': 'zowie', 'bar': { 'baz': ['giraffe'] }, 'quux': 'zip', 'quiggle': 'squiggle' } nested_composite_config_type = resolve_to_config_type({ 'fruts': { 'apple': Field(String), 'banana': Field(String, is_required=False), 'potato': Field(String, is_required=False, default_value='pie'), } }) with pytest.raises(CheckError, match='Missing non-optional composite member'): post_process_config(nested_composite_config_type, {'fruts': None}) with pytest.raises(CheckError, match='Missing non-optional composite member'): post_process_config(nested_composite_config_type, {'fruts': { 'banana': 'good', 'potato': 'bad' }}) assert post_process_config(nested_composite_config_type, { 'fruts': { 'apple': 'strawberry' } }).value == { 'fruts': { 'apple': 'strawberry', 'potato': 'pie' } } assert post_process_config(nested_composite_config_type, { 'fruts': { 'apple': 'a', 'banana': 'b', 'potato': 'c' } }).value == { 'fruts': { 'apple': 'a', 'banana': 'b', 'potato': 'c' } } any_config_type = resolve_to_config_type(Any) assert post_process_config(any_config_type, { 'foo': 'bar' }).value == { 'foo': 'bar' } assert post_process_config( ConfigType('gargle', given_name='bargle', kind=ConfigTypeKind.ANY), 3) selector_config_type = resolve_to_config_type( Selector({ 'one': Field(String), 'another': { 'foo': Field(String, default_value='bar', is_required=False) }, 'yet_another': Field(String, default_value='quux', is_required=False), })) with pytest.raises(CheckError): post_process_config(selector_config_type, 'one') with pytest.raises(ParameterCheckError): post_process_config(selector_config_type, None) with pytest.raises(ParameterCheckError, match='Expected dict with single item'): post_process_config(selector_config_type, {}) with pytest.raises(CheckError): post_process_config(selector_config_type, { 'one': 'foo', 'another': 'bar' }) assert post_process_config(selector_config_type, { 'one': 'foo' }).value == { 'one': 'foo' } assert post_process_config(selector_config_type, { 'one': None }).value == { 'one': None } assert post_process_config(selector_config_type, { 'one': {} }).value == { 'one': {} } assert post_process_config(selector_config_type, { 'another': {} }).value == { 'another': { 'foo': 'bar' } } singleton_selector_config_type = resolve_to_config_type( Selector( {'foo': Field(String, default_value='bar', is_required=False)})) assert post_process_config(singleton_selector_config_type, None).value == { 'foo': 'bar' } permissive_dict_config_type = resolve_to_config_type( Permissive({ 'foo': Field(String), 'bar': Field(String, default_value='baz', is_required=False) })) with pytest.raises(CheckError, match='Missing non-optional composite member'): post_process_config(permissive_dict_config_type, None) assert post_process_config(permissive_dict_config_type, { 'foo': 'wow', 'mau': 'mau' }).value == { 'foo': 'wow', 'bar': 'baz', 'mau': 'mau', }
def test_post_process_config(): scalar_config_type = resolve_to_config_type(String) assert post_process_config(scalar_config_type, "foo").value == "foo" assert post_process_config(scalar_config_type, 3).value == 3 assert post_process_config(scalar_config_type, {}).value == {} assert post_process_config(scalar_config_type, None).value is None enum_config_type = resolve_to_config_type( Enum("an_enum", [EnumValue("foo"), EnumValue("bar", python_value=3)])) assert post_process_config(enum_config_type, "foo").value == "foo" assert post_process_config(enum_config_type, "bar").value == 3 with pytest.raises(CheckError): post_process_config(enum_config_type, "baz") with pytest.raises(CheckError): post_process_config(enum_config_type, None) list_config_type = resolve_to_config_type([str]) assert post_process_config(list_config_type, ["foo"]).value == ["foo"] assert post_process_config(list_config_type, None).value == [] with pytest.raises(CheckError, match="Null array member not caught"): assert post_process_config(list_config_type, [None]).value == [None] nullable_list_config_type = resolve_to_config_type([Noneable(str)]) assert post_process_config(nullable_list_config_type, ["foo"]).value == ["foo"] assert post_process_config(nullable_list_config_type, [None]).value == [None] assert post_process_config(nullable_list_config_type, None).value == [] map_config_type = resolve_to_config_type({str: int}) assert post_process_config(map_config_type, {"foo": 5}).value == {"foo": 5} assert post_process_config(map_config_type, None).value == {} with pytest.raises(CheckError, match="Null map member not caught"): assert post_process_config(map_config_type, { "foo": None }).value == { "foo": None } nullable_map_config_type = resolve_to_config_type({str: Noneable(int)}) assert post_process_config(nullable_map_config_type, { "foo": 5 }).value == { "foo": 5 } assert post_process_config(nullable_map_config_type, { "foo": None }).value == { "foo": None } assert post_process_config(nullable_map_config_type, None).value == {} composite_config_type = resolve_to_config_type({ "foo": String, "bar": { "baz": [str] }, "quux": Field(str, is_required=False, default_value="zip"), "quiggle": Field(str, is_required=False), "werty": Field({str: [int]}, is_required=False), }) with pytest.raises(CheckError, match="Missing required composite member"): post_process_config(composite_config_type, {}) with pytest.raises(CheckError, match="Missing required composite member"): post_process_config(composite_config_type, { "bar": { "baz": ["giraffe"] }, "quux": "nimble" }) with pytest.raises(CheckError, match="Missing required composite member"): post_process_config(composite_config_type, { "foo": "zowie", "quux": "nimble" }) assert post_process_config(composite_config_type, { "foo": "zowie", "bar": { "baz": ["giraffe"] }, "quux": "nimble" }).value == { "foo": "zowie", "bar": { "baz": ["giraffe"] }, "quux": "nimble" } assert post_process_config(composite_config_type, { "foo": "zowie", "bar": { "baz": ["giraffe"] } }).value == { "foo": "zowie", "bar": { "baz": ["giraffe"] }, "quux": "zip" } assert post_process_config(composite_config_type, { "foo": "zowie", "bar": { "baz": ["giraffe"] }, "quiggle": "squiggle" }).value == { "foo": "zowie", "bar": { "baz": ["giraffe"] }, "quux": "zip", "quiggle": "squiggle" } assert post_process_config( composite_config_type, { "foo": "zowie", "bar": { "baz": ["giraffe"] }, "quiggle": "squiggle", "werty": { "asdf": [1, 2, 3] }, }, ).value == { "foo": "zowie", "bar": { "baz": ["giraffe"] }, "quux": "zip", "quiggle": "squiggle", "werty": { "asdf": [1, 2, 3] }, } nested_composite_config_type = resolve_to_config_type({ "fruts": { "apple": Field(String), "banana": Field(String, is_required=False), "potato": Field(String, is_required=False, default_value="pie"), } }) with pytest.raises(CheckError, match="Missing required composite member"): post_process_config(nested_composite_config_type, {"fruts": None}) with pytest.raises(CheckError, match="Missing required composite member"): post_process_config(nested_composite_config_type, {"fruts": { "banana": "good", "potato": "bad" }}) assert post_process_config(nested_composite_config_type, { "fruts": { "apple": "strawberry" } }).value == { "fruts": { "apple": "strawberry", "potato": "pie" } } assert post_process_config(nested_composite_config_type, { "fruts": { "apple": "a", "banana": "b", "potato": "c" } }).value == { "fruts": { "apple": "a", "banana": "b", "potato": "c" } } any_config_type = resolve_to_config_type(Any) assert post_process_config(any_config_type, { "foo": "bar" }).value == { "foo": "bar" } assert post_process_config( ConfigType("gargle", given_name="bargle", kind=ConfigTypeKind.ANY), 3) selector_config_type = resolve_to_config_type( Selector({ "one": Field(String), "another": { "foo": Field(String, default_value="bar", is_required=False) }, "yet_another": Field(String, default_value="quux", is_required=False), })) with pytest.raises(CheckError): post_process_config(selector_config_type, "one") with pytest.raises(ParameterCheckError): post_process_config(selector_config_type, None) with pytest.raises(ParameterCheckError, match="Expected dict with single item"): post_process_config(selector_config_type, {}) with pytest.raises(CheckError): post_process_config(selector_config_type, { "one": "foo", "another": "bar" }) assert post_process_config(selector_config_type, { "one": "foo" }).value == { "one": "foo" } assert post_process_config(selector_config_type, { "one": None }).value == { "one": None } assert post_process_config(selector_config_type, { "one": {} }).value == { "one": {} } assert post_process_config(selector_config_type, { "another": {} }).value == { "another": { "foo": "bar" } } singleton_selector_config_type = resolve_to_config_type( Selector( {"foo": Field(String, default_value="bar", is_required=False)})) assert post_process_config(singleton_selector_config_type, None).value == { "foo": "bar" } permissive_dict_config_type = resolve_to_config_type( Permissive({ "foo": Field(String), "bar": Field(String, default_value="baz", is_required=False) })) with pytest.raises(CheckError, match="Missing required composite member"): post_process_config(permissive_dict_config_type, None) assert post_process_config(permissive_dict_config_type, { "foo": "wow", "mau": "mau" }).value == { "foo": "wow", "bar": "baz", "mau": "mau", } noneable_permissive_config_type = resolve_to_config_type({ "args": Field(Noneable(Permissive()), is_required=False, default_value=None) }) assert post_process_config(noneable_permissive_config_type, { "args": { "foo": "wow", "mau": "mau" } }).value["args"] == { "foo": "wow", "mau": "mau", } assert post_process_config(noneable_permissive_config_type, { "args": {} }).value["args"] == {} assert post_process_config(noneable_permissive_config_type, None).value["args"] == None