def config_from_yaml_strings(yaml_strings):
"""Static constructor for run configs from YAML strings.
Args:
yaml_strings (List[str]): List of yaml strings to parse as the run config.
Returns:
Dict[Str, Any]: A run config dictionary constructed from the provided yaml strings
Raises:
DagsterInvariantViolationError: When one of the YAML documents is invalid and has a
parse error.
"""
yaml_strings = check.opt_list_param(yaml_strings,
"yaml_strings",
of_type=str)
try:
run_config = merge_yaml_strings(yaml_strings)
except yaml.YAMLError as err:
raise DagsterInvariantViolationError(
f"Encountered error attempting to parse yaml. Parsing YAMLs {yaml_strings} "
) from err
return run_config
def from_yaml_strings(name,
yaml_strings=None,
solid_selection=None,
mode=None,
tags=None):
'''Static constructor for presets from YAML strings.
Args:
name (str): The name of this preset. Must be unique in the presets defined on a given
pipeline.
yaml_strings (Optional[List[str]]): List of yaml strings to parse as the environment
config for this preset.
solid_selection (Optional[List[str]]): A list of solid subselection (including single
solid names) to execute with the preset. e.g. ``['*some_solid+', 'other_solid']``
mode (Optional[str]): The mode to apply when executing this preset. (default:
'default')
tags (Optional[Dict[str, Any]]): The tags to apply when executing this preset.
Returns:
PresetDefinition: A PresetDefinition constructed from the provided YAML strings
Raises:
DagsterInvariantViolationError: When one of the YAML documents is invalid and has a
parse error.
'''
check.str_param(name, 'name')
yaml_strings = check.opt_list_param(yaml_strings,
'yaml_strings',
of_type=str)
solid_selection = check.opt_nullable_list_param(solid_selection,
'solid_selection',
of_type=str)
mode = check.opt_str_param(mode, 'mode', DEFAULT_MODE_NAME)
try:
merged = merge_yaml_strings(yaml_strings)
except yaml.YAMLError as err:
six.raise_from(
DagsterInvariantViolationError(
'Encountered error attempting to parse yaml. Parsing YAMLs {yaml_strings} '
'on preset "{name}".'.format(yaml_strings=yaml_strings,
name=name)),
err,
)
return PresetDefinition(name, merged, solid_selection, mode, tags)
def test_merge_yaml_strings():
a = '''
foo:
bar: 1
baz: 3
'''
b = '''
foo:
one: "one"
other: 4
'''
c = '''
final: "result"
'''
assert merge_yaml_strings([a, b, c]) == {
'final': 'result',
'foo': {
'one': 'one',
'bar': 1
},
'other': 4,
'baz': 3,
}
override = '''final: "some other result"'''
assert merge_yaml_strings([a, b, c, override]) == {
'foo': {
'bar': 1,
'one': 'one'
},
'baz': 3,
'other': 4,
'final': 'some other result',
}
string_yaml = 'this is a valid YAML string but not a dictionary'
expected = 'Expected YAML dictionary, instead got: "{string_yaml}"'.format(
string_yaml=string_yaml)
with pytest.raises(check.CheckError, match=expected):
merge_yaml_strings([a, string_yaml])
with pytest.raises(
yaml.YAMLError,
match=
"while scanning for the next token\nfound character '`' that cannot start any token",
):
bad_yaml = '--- `'
merge_yaml_strings([a, bad_yaml])
def test_merge_yaml_strings():
a = """
foo:
bar: 1
baz: 3
"""
b = """
foo:
one: "one"
other: 4
"""
c = """
final: "result"
"""
assert merge_yaml_strings([a, b, c]) == {
"final": "result",
"foo": {
"one": "one",
"bar": 1
},
"other": 4,
"baz": 3,
}
override = '''final: "some other result"'''
assert merge_yaml_strings([a, b, c, override]) == {
"foo": {
"bar": 1,
"one": "one"
},
"baz": 3,
"other": 4,
"final": "some other result",
}
string_yaml = "this is a valid YAML string but not a dictionary"
expected = 'Expected YAML dictionary, instead got: "{string_yaml}"'.format(
string_yaml=string_yaml)
with pytest.raises(check.CheckError, match=expected):
merge_yaml_strings([a, string_yaml])
with pytest.raises(
yaml.YAMLError,
match=
"while scanning for the next token\nfound character '`' that cannot start any token",
):
bad_yaml = "--- `"
merge_yaml_strings([a, bad_yaml])
