def config_from_yaml_strings(yaml_strings): """Static constructor for run configs from YAML strings. Args: yaml_strings (List[str]): List of yaml strings to parse as the run config. Returns: Dict[Str, Any]: A run config dictionary constructed from the provided yaml strings Raises: DagsterInvariantViolationError: When one of the YAML documents is invalid and has a parse error. """ yaml_strings = check.opt_list_param(yaml_strings, "yaml_strings", of_type=str) try: run_config = merge_yaml_strings(yaml_strings) except yaml.YAMLError as err: raise DagsterInvariantViolationError( f"Encountered error attempting to parse yaml. Parsing YAMLs {yaml_strings} " ) from err return run_config
def from_yaml_strings(name, yaml_strings=None, solid_selection=None, mode=None, tags=None): '''Static constructor for presets from YAML strings. Args: name (str): The name of this preset. Must be unique in the presets defined on a given pipeline. yaml_strings (Optional[List[str]]): List of yaml strings to parse as the environment config for this preset. solid_selection (Optional[List[str]]): A list of solid subselection (including single solid names) to execute with the preset. e.g. ``['*some_solid+', 'other_solid']`` mode (Optional[str]): The mode to apply when executing this preset. (default: 'default') tags (Optional[Dict[str, Any]]): The tags to apply when executing this preset. Returns: PresetDefinition: A PresetDefinition constructed from the provided YAML strings Raises: DagsterInvariantViolationError: When one of the YAML documents is invalid and has a parse error. ''' check.str_param(name, 'name') yaml_strings = check.opt_list_param(yaml_strings, 'yaml_strings', of_type=str) solid_selection = check.opt_nullable_list_param(solid_selection, 'solid_selection', of_type=str) mode = check.opt_str_param(mode, 'mode', DEFAULT_MODE_NAME) try: merged = merge_yaml_strings(yaml_strings) except yaml.YAMLError as err: six.raise_from( DagsterInvariantViolationError( 'Encountered error attempting to parse yaml. Parsing YAMLs {yaml_strings} ' 'on preset "{name}".'.format(yaml_strings=yaml_strings, name=name)), err, ) return PresetDefinition(name, merged, solid_selection, mode, tags)
def test_merge_yaml_strings(): a = ''' foo: bar: 1 baz: 3 ''' b = ''' foo: one: "one" other: 4 ''' c = ''' final: "result" ''' assert merge_yaml_strings([a, b, c]) == { 'final': 'result', 'foo': { 'one': 'one', 'bar': 1 }, 'other': 4, 'baz': 3, } override = '''final: "some other result"''' assert merge_yaml_strings([a, b, c, override]) == { 'foo': { 'bar': 1, 'one': 'one' }, 'baz': 3, 'other': 4, 'final': 'some other result', } string_yaml = 'this is a valid YAML string but not a dictionary' expected = 'Expected YAML dictionary, instead got: "{string_yaml}"'.format( string_yaml=string_yaml) with pytest.raises(check.CheckError, match=expected): merge_yaml_strings([a, string_yaml]) with pytest.raises( yaml.YAMLError, match= "while scanning for the next token\nfound character '`' that cannot start any token", ): bad_yaml = '--- `' merge_yaml_strings([a, bad_yaml])
def test_merge_yaml_strings(): a = """ foo: bar: 1 baz: 3 """ b = """ foo: one: "one" other: 4 """ c = """ final: "result" """ assert merge_yaml_strings([a, b, c]) == { "final": "result", "foo": { "one": "one", "bar": 1 }, "other": 4, "baz": 3, } override = '''final: "some other result"''' assert merge_yaml_strings([a, b, c, override]) == { "foo": { "bar": 1, "one": "one" }, "baz": 3, "other": 4, "final": "some other result", } string_yaml = "this is a valid YAML string but not a dictionary" expected = 'Expected YAML dictionary, instead got: "{string_yaml}"'.format( string_yaml=string_yaml) with pytest.raises(check.CheckError, match=expected): merge_yaml_strings([a, string_yaml]) with pytest.raises( yaml.YAMLError, match= "while scanning for the next token\nfound character '`' that cannot start any token", ): bad_yaml = "--- `" merge_yaml_strings([a, bad_yaml])