def _create_config_loader(self, conf_paths: Iterable[str]) -> TemplatedConfigLoader: return TemplatedConfigLoader( conf_paths, globals_pattern="*globals.yml", # read the globals dictionary from project config globals_dict={ # extra keys to add to the globals dictionary, take precedence over globals_pattern "bucket_name": "another_bucket_name", "non_string_key": 10, }, )
def test_catlog_parameterized_exceptional(self, tmp_path, conf_paths, template_config_exceptional): """Test templating with mixed type replacement values going into one string""" (tmp_path / "local").mkdir(exist_ok=True) catalog = TemplatedConfigLoader( conf_paths, globals_dict=template_config_exceptional).get("catalog*.yml") assert catalog["postcode"] == "NW10 2JK"
def test_catlog_parameterized_no_params(self, tmp_path, conf_paths): """Test parameterized config without input""" (tmp_path / "local").mkdir(exist_ok=True) catalog = TemplatedConfigLoader(conf_paths).get("catalog*.yml") assert catalog["boats"]["type"] == "${boat_data_type}" assert (catalog["boats"]["filepath"] == "${s3_bucket}/${raw_data_folder}/${boat_file_name}") assert catalog["boats"]["columns"]["id"] == "${string_type}" assert catalog["boats"]["columns"]["name"] == "${string_type}" assert catalog["boats"]["columns"]["top_speed"] == "${float_type}" assert catalog["boats"]["users"] == ["fred", "${write_only_user}"]
def test_catlog_parameterized_w_globals(self, tmp_path, conf_paths): """Test parameterized config with globals yaml file""" (tmp_path / "local").mkdir(exist_ok=True) catalog = TemplatedConfigLoader( conf_paths, globals_pattern="*globals.yml").get("catalog*.yml") assert catalog["boats"]["type"] == "SparkDataSet" assert (catalog["boats"]["filepath"] == "s3a://boat-and-car-bucket/01_raw/boats.csv") assert catalog["boats"]["columns"]["id"] == "VARCHAR" assert catalog["boats"]["columns"]["name"] == "VARCHAR" assert catalog["boats"]["columns"]["top_speed"] == "FLOAT" assert catalog["boats"]["users"] == ["fred", "ron"]
def test_catlog_advanced(self, tmp_path, conf_paths, normal_config_advanced): """Test whether it responds well to advanced yaml values (i.e. nested dicts, booleans, lists, etc.)""" (tmp_path / "local").mkdir(exist_ok=True) catalog = TemplatedConfigLoader( conf_paths, globals_dict=normal_config_advanced ).get("catalog*.yml") assert catalog["planes"]["type"] == "SparkJDBCDataSet" assert catalog["planes"]["postgres_credentials"]["user"] == "Fakeuser" assert catalog["planes"]["postgres_credentials"]["password"] == "F@keP@55word" assert catalog["planes"]["batch_size"] == 10000 assert catalog["planes"]["need_permission"] assert catalog["planes"]["secret_tables"] == ["models", "pilots", "engines"]
def test_catlog_parameterized_w_dict_namespaced( self, tmp_path, conf_paths, template_config, get_environ ): """Test parameterized config with namespacing in the template values""" (tmp_path / "local").mkdir(exist_ok=True) catalog = TemplatedConfigLoader( conf_paths, globals_dict={"global": template_config, "env": get_environ} ).get("catalog*.yml") assert catalog["boats"]["type"] == "SparkDataSet" assert ( catalog["boats"]["filepath"] == "s3a://boat-and-car-bucket/01_raw/boats.csv" ) assert catalog["boats"]["columns"]["id"] == "VARCHAR" assert catalog["boats"]["columns"]["name"] == "VARCHAR" assert catalog["boats"]["columns"]["top_speed"] == "FLOAT" assert catalog["boats"]["users"] == ["fred", "ron"]
def _create_config_loader(self, conf_paths: Iterable[str]) -> TemplatedConfigLoader: return TemplatedConfigLoader(conf_paths, globals_pattern="*globals.yml",)