def _create_config_loader(self, conf_paths: Iterable[str]) -> TemplatedConfigLoader:
return TemplatedConfigLoader(
conf_paths,
globals_pattern="*globals.yml", # read the globals dictionary from project config
globals_dict={ # extra keys to add to the globals dictionary, take precedence over globals_pattern
"bucket_name": "another_bucket_name",
"non_string_key": 10,
},
)
def test_catlog_parameterized_exceptional(self, tmp_path, conf_paths,
template_config_exceptional):
"""Test templating with mixed type replacement values going into one string"""
(tmp_path / "local").mkdir(exist_ok=True)
catalog = TemplatedConfigLoader(
conf_paths,
globals_dict=template_config_exceptional).get("catalog*.yml")
assert catalog["postcode"] == "NW10 2JK"
def test_catlog_parameterized_no_params(self, tmp_path, conf_paths):
"""Test parameterized config without input"""
(tmp_path / "local").mkdir(exist_ok=True)
catalog = TemplatedConfigLoader(conf_paths).get("catalog*.yml")
assert catalog["boats"]["type"] == "${boat_data_type}"
assert (catalog["boats"]["filepath"] ==
"${s3_bucket}/${raw_data_folder}/${boat_file_name}")
assert catalog["boats"]["columns"]["id"] == "${string_type}"
assert catalog["boats"]["columns"]["name"] == "${string_type}"
assert catalog["boats"]["columns"]["top_speed"] == "${float_type}"
assert catalog["boats"]["users"] == ["fred", "${write_only_user}"]
def test_catlog_parameterized_w_globals(self, tmp_path, conf_paths):
"""Test parameterized config with globals yaml file"""
(tmp_path / "local").mkdir(exist_ok=True)
catalog = TemplatedConfigLoader(
conf_paths, globals_pattern="*globals.yml").get("catalog*.yml")
assert catalog["boats"]["type"] == "SparkDataSet"
assert (catalog["boats"]["filepath"] ==
"s3a://boat-and-car-bucket/01_raw/boats.csv")
assert catalog["boats"]["columns"]["id"] == "VARCHAR"
assert catalog["boats"]["columns"]["name"] == "VARCHAR"
assert catalog["boats"]["columns"]["top_speed"] == "FLOAT"
assert catalog["boats"]["users"] == ["fred", "ron"]
def test_catlog_advanced(self, tmp_path, conf_paths, normal_config_advanced):
"""Test whether it responds well to advanced yaml values
(i.e. nested dicts, booleans, lists, etc.)"""
(tmp_path / "local").mkdir(exist_ok=True)
catalog = TemplatedConfigLoader(
conf_paths, globals_dict=normal_config_advanced
).get("catalog*.yml")
assert catalog["planes"]["type"] == "SparkJDBCDataSet"
assert catalog["planes"]["postgres_credentials"]["user"] == "Fakeuser"
assert catalog["planes"]["postgres_credentials"]["password"] == "F@keP@55word"
assert catalog["planes"]["batch_size"] == 10000
assert catalog["planes"]["need_permission"]
assert catalog["planes"]["secret_tables"] == ["models", "pilots", "engines"]
def test_catlog_parameterized_w_dict_namespaced(
self, tmp_path, conf_paths, template_config, get_environ
):
"""Test parameterized config with namespacing in the template values"""
(tmp_path / "local").mkdir(exist_ok=True)
catalog = TemplatedConfigLoader(
conf_paths, globals_dict={"global": template_config, "env": get_environ}
).get("catalog*.yml")
assert catalog["boats"]["type"] == "SparkDataSet"
assert (
catalog["boats"]["filepath"] == "s3a://boat-and-car-bucket/01_raw/boats.csv"
)
assert catalog["boats"]["columns"]["id"] == "VARCHAR"
assert catalog["boats"]["columns"]["name"] == "VARCHAR"
assert catalog["boats"]["columns"]["top_speed"] == "FLOAT"
assert catalog["boats"]["users"] == ["fred", "ron"]
def _create_config_loader(self, conf_paths: Iterable[str]) -> TemplatedConfigLoader:
return TemplatedConfigLoader(conf_paths, globals_pattern="*globals.yml",)